diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce0cfef9e..0d9a85651 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,3 +40,23 @@ jobs: - name: Test run: npm test + + mcp-conformance: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + cache: npm + + - name: Install dependencies + run: npm ci --ignore-scripts + + # Runs pi's MCP client from source against the pinned official conformance suite and fails on + # regressions against packages/coding-agent/test/mcp-conformance/baseline.json. + - name: MCP client conformance + run: npm run test:mcp-conformance diff --git a/package.json b/package.json index 26493c596..5ef19c64a 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "profile:rpc": "node scripts/profile-coding-agent-node.mjs --mode rpc", "test": "npm run test:scripts && npm run test --workspaces --if-present", "test:scripts": "node --test scripts/*.test.mjs scripts/*.test.ts", + "test:mcp-conformance": "node packages/coding-agent/test/mcp-conformance/run.ts", "version:patch": "npm version patch --workspaces --no-git-tag-version --no-workspaces-update && node scripts/sync-versions.js && npm install --package-lock-only --ignore-scripts", "version:minor": "npm version minor --workspaces --no-git-tag-version --no-workspaces-update && node scripts/sync-versions.js && npm install --package-lock-only --ignore-scripts", "version:major": "npm version major --workspaces --no-git-tag-version --no-workspaces-update && node scripts/sync-versions.js && npm install --package-lock-only --ignore-scripts", diff --git a/packages/coding-agent/test/mcp-conformance/README.md b/packages/coding-agent/test/mcp-conformance/README.md new file mode 100644 index 000000000..8aaa36690 --- /dev/null +++ b/packages/coding-agent/test/mcp-conformance/README.md @@ -0,0 +1,45 @@ +# MCP client conformance + +Runs the official [MCP conformance suite](https://github.com/modelcontextprotocol/conformance) against pi's MCP client and compares the result with a committed baseline. + +```bash +npm run test:mcp-conformance +``` + +CI runs it in the `mcp-conformance` job of `.github/workflows/ci.yml`. Requires network access the first time, to fetch the pinned suite (`@modelcontextprotocol/conformance@0.2.0-alpha.11`) with `npx`. Install scripts are disabled. Needs a POSIX shell. + +## How it works + +For every protocol version pi negotiates (`2025-03-26`, `2025-06-18`, `2025-11-25`), `run.ts` lists the suite's client scenarios and runs them one at a time. The suite starts a scenario server and runs `client.ts` against it. + +`client.ts` uses the code pi runs for an HTTP server in `mcp.json`: `McpServerConnection` connects and calls tools, and `signInMcpServer` runs the OAuth sign-in that `/mcp` starts. A simulated browser fetches the authorization URL and delivers the redirect to pi's loopback callback server. When the server asks for sign-in again (for example for more scope), the simulated user signs in again, up to three times. + +Every check the suite reports is compared with `baseline.json`. The extra `pi-client` check records whether `client.ts` completed the scenario. Some scenarios expect the client to give up (`auth/scope-retry-limit`), so it is baselined like the others. + +The run fails when: + +- a check that passes in the baseline fails or is missing, +- a check fails that the baseline does not list as failing, +- a baselined scenario did not run. + +Checks that started passing are reported; update the baseline to lock them in. + +## Options + +```bash +node packages/coding-agent/test/mcp-conformance/run.ts --mode 2025-11-25 --scenario auth/scope-step-up --verbose +node packages/coding-agent/test/mcp-conformance/run.ts --update-baseline +``` + +- `--mode`, `--scenario`: run a subset (repeatable). +- `--verbose`: print the suite's output, including each check and the client's log. +- `--keep-results`: keep the suite's `checks.json` and client output. +- `--update-baseline`: write the results of a full run to `baseline.json`. Review the diff; do not regenerate it to hide a regression. + +## Known failures + +- `elicitation-sep1034-client-defaults`: pi does not support elicitation. +- `auth/basic-cimd`: pi registers clients dynamically instead of using a Client ID Metadata Document. +- `auth/scope-retry-limit`: `pi-client` fails by design; the server never accepts the granted scope. + +2026-07-28 is not covered: it is a stateless protocol pi does not implement. diff --git a/packages/coding-agent/test/mcp-conformance/baseline.json b/packages/coding-agent/test/mcp-conformance/baseline.json new file mode 100644 index 000000000..0401d64b0 --- /dev/null +++ b/packages/coding-agent/test/mcp-conformance/baseline.json @@ -0,0 +1,413 @@ +{ + "conformance": "@modelcontextprotocol/conformance@0.2.0-alpha.11", + "modes": { + "2025-03-26": { + "auth/2025-03-26-oauth-metadata-backcompat": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "token-request", + "valid-bearer-token" + ], + "failing": [] + }, + "auth/2025-03-26-oauth-endpoint-fallback": { + "passing": [ + "authorization-request", + "client-registration", + "pi-client", + "token-request", + "valid-bearer-token" + ], + "failing": [] + } + }, + "2025-06-18": { + "initialize": { + "passing": [ + "mcp-client-initialization", + "pi-client" + ], + "failing": [] + }, + "tools_call": { + "passing": [ + "pi-client", + "tool-add-numbers", + "wire-schema-valid" + ], + "failing": [] + }, + "auth/token-endpoint-auth-basic": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "prm-pathbased-requested", + "resource-parameter-consistency", + "resource-parameter-in-authorization", + "resource-parameter-in-token", + "resource-parameter-valid-uri", + "token-endpoint-auth-method", + "token-request", + "valid-bearer-token" + ], + "failing": [] + }, + "auth/token-endpoint-auth-post": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "prm-pathbased-requested", + "resource-parameter-consistency", + "resource-parameter-in-authorization", + "resource-parameter-in-token", + "resource-parameter-valid-uri", + "token-endpoint-auth-method", + "token-request", + "valid-bearer-token" + ], + "failing": [] + }, + "auth/token-endpoint-auth-none": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "prm-pathbased-requested", + "resource-parameter-consistency", + "resource-parameter-in-authorization", + "resource-parameter-in-token", + "resource-parameter-valid-uri", + "token-endpoint-auth-method", + "token-request", + "valid-bearer-token" + ], + "failing": [] + } + }, + "2025-11-25": { + "initialize": { + "passing": [ + "mcp-client-initialization", + "pi-client" + ], + "failing": [] + }, + "tools_call": { + "passing": [ + "pi-client", + "tool-add-numbers", + "wire-schema-valid" + ], + "failing": [] + }, + "elicitation-sep1034-client-defaults": { + "passing": [ + "pi-client" + ], + "failing": [ + "client-elicitation-sep1034-boolean-default", + "client-elicitation-sep1034-enum-default", + "client-elicitation-sep1034-general", + "client-elicitation-sep1034-integer-default", + "client-elicitation-sep1034-number-default", + "client-elicitation-sep1034-string-default" + ] + }, + "sse-retry": { + "passing": [ + "client-sse-graceful-reconnect", + "client-sse-last-event-id", + "client-sse-retry-timing", + "pi-client" + ], + "failing": [] + }, + "auth/metadata-default": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "prm-pathbased-requested", + "token-request", + "valid-bearer-token" + ], + "failing": [] + }, + "auth/metadata-var1": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "prm-pathbased-requested", + "token-request", + "valid-bearer-token" + ], + "failing": [] + }, + "auth/metadata-var2": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "prm-pathbased-requested", + "token-request", + "valid-bearer-token" + ], + "failing": [] + }, + "auth/metadata-var3": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "prm-pathbased-requested", + "token-request", + "valid-bearer-token" + ], + "failing": [] + }, + "auth/basic-cimd": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "prm-pathbased-requested", + "token-request", + "valid-bearer-token" + ], + "failing": [ + "cimd-client-id-used" + ] + }, + "auth/scope-from-www-authenticate": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "prm-pathbased-requested", + "scope-from-www-authenticate", + "token-request", + "valid-bearer-token" + ], + "failing": [] + }, + "auth/scope-from-scopes-supported": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "prm-pathbased-requested", + "scope-from-scopes-supported", + "token-request", + "valid-bearer-token" + ], + "failing": [] + }, + "auth/scope-omitted-when-undefined": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "prm-pathbased-requested", + "scope-omitted-when-undefined", + "token-request", + "valid-bearer-token" + ], + "failing": [] + }, + "auth/scope-step-up": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "prm-pathbased-requested", + "scope-step-up-escalation", + "scope-step-up-initial", + "token-request", + "valid-bearer-token" + ], + "failing": [] + }, + "auth/scope-retry-limit": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "prm-pathbased-requested", + "scope-retry-limit", + "token-request" + ], + "failing": [ + "pi-client" + ] + }, + "auth/token-endpoint-auth-basic": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "prm-pathbased-requested", + "resource-parameter-consistency", + "resource-parameter-in-authorization", + "resource-parameter-in-token", + "resource-parameter-valid-uri", + "token-endpoint-auth-method", + "token-request", + "valid-bearer-token" + ], + "failing": [] + }, + "auth/token-endpoint-auth-post": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "prm-pathbased-requested", + "resource-parameter-consistency", + "resource-parameter-in-authorization", + "resource-parameter-in-token", + "resource-parameter-valid-uri", + "token-endpoint-auth-method", + "token-request", + "valid-bearer-token" + ], + "failing": [] + }, + "auth/token-endpoint-auth-none": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "client-registration", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "prm-pathbased-requested", + "resource-parameter-consistency", + "resource-parameter-in-authorization", + "resource-parameter-in-token", + "resource-parameter-valid-uri", + "token-endpoint-auth-method", + "token-request", + "valid-bearer-token" + ], + "failing": [] + }, + "auth/pre-registration": { + "passing": [ + "authorization-request", + "authorization-server-metadata", + "pi-client", + "pkce-code-challenge-sent", + "pkce-code-verifier-sent", + "pkce-s256-method-used", + "pkce-verifier-matches-challenge", + "pre-registration-auth", + "prm-pathbased-requested", + "token-request", + "valid-bearer-token" + ], + "failing": [] + }, + "json-schema-2020-12-preservation": { + "passing": [ + "json-schema-2020-12-client-$defs-preserved", + "json-schema-2020-12-client-$schema-preserved", + "json-schema-2020-12-client-additionalProperties-preserved", + "json-schema-2020-12-client-echo-completed", + "json-schema-2020-12-client-tool-found", + "pi-client", + "sep-2106-client-anchor-keyword-preserved", + "sep-2106-client-composition-keywords-preserved", + "sep-2106-client-conditional-keywords-preserved", + "wire-schema-valid" + ], + "failing": [] + } + } + } +} diff --git a/packages/coding-agent/test/mcp-conformance/client.ts b/packages/coding-agent/test/mcp-conformance/client.ts new file mode 100644 index 000000000..cea10ced2 --- /dev/null +++ b/packages/coding-agent/test/mcp-conformance/client.ts @@ -0,0 +1,200 @@ +/** + * Client under test for the official MCP client conformance suite. The suite starts a scenario server + * and runs this script with the server URL as the last argument, the scenario name in + * `MCP_CONFORMANCE_SCENARIO`, and scenario data in `MCP_CONFORMANCE_CONTEXT`. + * + * It drives the code pi runs for an `mcp.json` HTTP server: `McpServerConnection` connects and calls + * tools, and `signInMcpServer` runs the OAuth sign-in that `/mcp` starts. The browser is simulated: the + * authorization URL is fetched without following its redirect, and the redirect is delivered to pi's + * loopback callback server. Credentials stay in memory. + * + * Run through run.ts, which writes the outcome to `PI_MCP_CONFORMANCE_REPORT`. + */ + +import { writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { InMemoryAuthStorageBackend } from "../../src/core/auth-storage.ts"; +import type { McpHttpServerConfig, McpServerEntry } from "../../src/extensions/mcp/config.ts"; +import type { McpSignInPrompt } from "../../src/extensions/mcp/oauth.ts"; +import { + createDefaultTransport, + McpOAuthCredentialStore, + McpServerConnection, + signInMcpServer, +} from "../../src/extensions/mcp/runtime.ts"; + +/** + * Sign-ins pi asks the user for are not limited, so a user who keeps approving them would loop + * forever against a server that never accepts the granted scope. This simulated user gives up after + * three, the limit `auth/scope-retry-limit` checks. + */ +const MAX_SIGN_INS = 3; +const REQUEST_TIMEOUT_SECONDS = 20; + +interface ToolCall { + name: string; + arguments: Record; +} + +/** Tool calls of a scenario after connecting. `undefined` for scenarios this client does not know. */ +function toolCalls(scenario: string, connection: McpServerConnection): ToolCall[] | undefined { + if (scenario.startsWith("auth/")) return [{ name: "test-tool", arguments: {} }]; + switch (scenario) { + case "initialize": + return []; + case "tools_call": + return [{ name: "add_numbers", arguments: { a: 2, b: 3 } }]; + case "sse-retry": + return [{ name: "test_reconnection", arguments: {} }]; + case "elicitation-sep1034-client-defaults": + return [{ name: "test_client_elicitation_defaults", arguments: {} }]; + case "json-schema-2020-12-preservation": { + // The server compares the echoed schema with the one it listed, to detect dropped keywords. + const tool = connection.tools.find((candidate) => candidate.name === "json_schema_2020_12_tool"); + if (!tool) throw new Error("json_schema_2020_12_tool was not listed"); + return [{ name: "json_schema_echo", arguments: { schema: tool.inputSchema } }]; + } + default: + return undefined; + } +} + +function log(message: string): void { + process.stderr.write(`[pi-conformance] ${message}\n`); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function readContext(): Record { + const raw = process.env.MCP_CONFORMANCE_CONTEXT; + if (!raw) return {}; + const parsed: unknown = JSON.parse(raw); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw new Error("MCP_CONFORMANCE_CONTEXT is not an object"); + } + return parsed as Record; +} + +function isLoopback(url: URL): boolean { + return ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname); +} + +/** What a browser does with the authorization URL when the user approves at once. */ +async function visitAuthorizationUrl(url: URL): Promise { + const response = await fetch(url, { redirect: "manual" }); + const location = response.headers.get("location"); + if (!location) throw new Error(`Authorization endpoint answered ${response.status} without a redirect`); + const callback = new URL(location, url); + if (callback.protocol !== "http:" || !isLoopback(callback)) { + throw new Error(`Authorization endpoint redirected to ${callback.origin}, not to the loopback callback`); + } + log(`delivering authorization response to ${callback.origin}${callback.pathname}`); + await (await fetch(callback)).text(); +} + +function simulatedBrowser(): McpSignInPrompt { + let visit: Promise | undefined; + return { + showAuthorizationUrl(url) { + log(`authorization URL: ${url.origin}${url.pathname}`); + visit = visitAuthorizationUrl(url); + }, + async promptForRedirectUrl(signal) { + try { + await visit; + } catch (error) { + // Cancels the sign-in instead of waiting for a callback that never comes. + log(`browser failed: ${errorMessage(error)}`); + return undefined; + } + if (!signal.aborted) { + await new Promise((resolve) => signal.addEventListener("abort", resolve, { once: true })); + } + return undefined; + }, + }; +} + +async function run(serverUrl: string, scenario: string): Promise { + const context = readContext(); + const config: McpHttpServerConfig = { url: serverUrl, timeout: REQUEST_TIMEOUT_SECONDS }; + if (typeof context.client_id === "string") { + // Pre-registered clients are configured in `mcp.json`. + config.oauth = { + clientId: context.client_id, + clientSecret: typeof context.client_secret === "string" ? context.client_secret : undefined, + }; + } + const entry: McpServerEntry = { name: "conformance", config, source: "conformance" }; + const credentials = new McpOAuthCredentialStore(new InMemoryAuthStorageBackend()); + const connection = new McpServerConnection({ + entry, + cwd: tmpdir(), + createTransport: createDefaultTransport, + credentials, + onTools: () => {}, + }); + + let signIns = 0; + /** Run `operation`, signing in like a user answering `/mcp` whenever the server asks for it. */ + const withSignIn = async (operation: () => Promise): Promise => { + for (;;) { + try { + return await operation(); + } catch (error) { + const url = connection.oauthUrl; + if (connection.state !== "needs-auth" || !url) throw error; + if (signIns >= MAX_SIGN_INS) throw new Error(`Still requires sign-in after ${signIns} sign-ins`); + signIns++; + log(`sign-in ${signIns}${connection.challenge?.scope ? ` (scope: ${connection.challenge.scope})` : ""}`); + await signInMcpServer({ + serverUrl: url, + store: credentials.forServer(url), + settings: connection.oauthSettings(), + challenge: connection.challenge, + prompt: simulatedBrowser(), + }); + connection.challenge = undefined; + } + } + }; + + try { + await withSignIn(() => connection.getClient()); + log(`connected, tools: ${connection.tools.map((tool) => tool.name).join(", ") || "(none)"}`); + const calls = toolCalls(scenario, connection); + if (!calls) throw new Error(`Unknown scenario ${scenario}`); + for (const call of calls) { + const result = await withSignIn(() => connection.callTool(call.name, call.arguments, {})); + log(`${call.name}: ${JSON.stringify(result.content)}`); + if (result.isError) throw new Error(`Tool ${call.name} failed: ${JSON.stringify(result.content)}`); + } + } finally { + await connection.close(); + } +} + +async function main(): Promise { + const serverUrl = process.argv.at(-1); + const scenario = process.env.MCP_CONFORMANCE_SCENARIO; + const reportPath = process.env.PI_MCP_CONFORMANCE_REPORT; + let report: { success: boolean; error?: string }; + if (!serverUrl || !scenario || process.argv.length < 3) { + report = { success: false, error: "Usage: MCP_CONFORMANCE_SCENARIO= client.ts " }; + } else { + try { + await run(serverUrl, scenario); + report = { success: true }; + } catch (error) { + report = { success: false, error: errorMessage(error) }; + } + } + if (report.error) log(`error: ${report.error}`); + if (reportPath) writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`); + return report.success ? 0 : 1; +} + +// Exit explicitly: a failed scenario can leave sockets of the server under test open. +process.exit(await main()); diff --git a/packages/coding-agent/test/mcp-conformance/run.ts b/packages/coding-agent/test/mcp-conformance/run.ts new file mode 100644 index 000000000..365d28cbd --- /dev/null +++ b/packages/coding-agent/test/mcp-conformance/run.ts @@ -0,0 +1,365 @@ +/** + * Runs the official MCP client conformance suite against pi (client.ts) and compares every check with + * the committed baseline (baseline.json). Fails when a check that passed in the baseline no longer + * passes, or when a check fails that the baseline does not list. Known failures stay visible in the + * output. See README.md. + */ + +import { spawn } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { parseArgs } from "node:util"; + +/** The pinned upstream suite. Bumping it requires reviewing and regenerating the baseline. */ +const CONFORMANCE_PACKAGE = "@modelcontextprotocol/conformance@0.2.0-alpha.11"; +/** Protocol versions pi negotiates. 2026-07-28 is a different (stateless) protocol pi does not speak. */ +const MODES = ["2025-03-26", "2025-06-18", "2025-11-25"] as const; +const SCENARIO_TIMEOUT_MS = 30_000; +/** Checks that are part of the suite's HTTP trace, not assertions. */ +const IGNORED_STATUSES = new Set(["INFO", "SKIPPED"]); + +const here = fileURLToPath(new URL(".", import.meta.url)); +const baselinePath = join(here, "baseline.json"); +const clientPath = join(here, "client.ts"); +const resolverUrl = pathToFileURL(join(here, "../../src/experimental/source-resolver.ts")).href; + +type CheckStatus = "pass" | "fail"; + +interface ScenarioResult { + mode: string; + scenario: string; + /** Check id to status. Repeated ids pass only when every instance passes. */ + checks: Map; + /** Failure messages by check id, for the report. */ + messages: Map; +} + +interface BaselineScenario { + passing: string[]; + failing: string[]; +} + +interface Baseline { + conformance: string; + modes: Record>; +} + +interface RawCheck { + id?: unknown; + name?: unknown; + status?: unknown; + description?: unknown; + errorMessage?: unknown; +} + +function runCommand( + command: string, + args: string[], + options: { cwd: string; env?: NodeJS.ProcessEnv; timeoutMs?: number }, +): Promise<{ code: number; stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { cwd: options.cwd, env: options.env ?? process.env, detached: true }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (data: Buffer) => { + stdout += data.toString(); + }); + child.stderr.on("data", (data: Buffer) => { + stderr += data.toString(); + }); + const timer = + options.timeoutMs === undefined + ? undefined + : setTimeout(() => { + // The runner spawns the client and scenario servers; end the whole group. + if (child.pid !== undefined) { + try { + process.kill(-child.pid, "SIGKILL"); + } catch {} + } + }, options.timeoutMs); + child.on("error", reject); + child.on("close", (code, signal) => { + if (timer) clearTimeout(timer); + resolve({ code: code ?? (signal ? 128 : 1), stdout, stderr }); + }); + }); +} + +function conformanceEnv(): NodeJS.ProcessEnv { + // The suite comes from npm with prebuilt files; nothing needs install scripts. + return { ...process.env, npm_config_ignore_scripts: "true", npm_config_yes: "true" }; +} + +async function listScenarios(mode: string, workDir: string): Promise { + const result = await runCommand("npx", ["--yes", CONFORMANCE_PACKAGE, "list", "--client", "--spec-version", mode], { + cwd: workDir, + env: conformanceEnv(), + }); + if (result.code !== 0) throw new Error(`Listing ${mode} scenarios failed:\n${result.stderr || result.stdout}`); + return [...result.stdout.matchAll(/^\s+- (\S+)/gm)].map((match) => match[1]); +} + +function findFile(dir: string, name: string): string | undefined { + if (!existsSync(dir)) return undefined; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isFile() && entry.name === name) return path; + if (entry.isDirectory()) { + const found = findFile(path, name); + if (found) return found; + } + } + return undefined; +} + +function record(result: ScenarioResult, id: string, status: CheckStatus, message?: string): void { + if (result.checks.get(id) !== "fail") result.checks.set(id, status); + if (status === "fail" && message && !result.messages.has(id)) result.messages.set(id, message); +} + +async function runScenario( + mode: string, + scenario: string, + launcher: string, + workDir: string, + verbose: boolean, +): Promise { + const scenarioDir = join(workDir, mode, scenario.replace(/[^A-Za-z0-9_.-]+/g, "-")); + mkdirSync(scenarioDir, { recursive: true }); + const reportPath = join(scenarioDir, "pi-client.json"); + const result: ScenarioResult = { mode, scenario, checks: new Map(), messages: new Map() }; + const run = await runCommand( + "npx", + [ + "--yes", + CONFORMANCE_PACKAGE, + "client", + "--command", + launcher, + "--scenario", + scenario, + "--spec-version", + mode, + "--timeout", + String(SCENARIO_TIMEOUT_MS), + "--output-dir", + scenarioDir, + ], + { + cwd: workDir, + env: { ...conformanceEnv(), PI_MCP_CONFORMANCE_REPORT: reportPath }, + timeoutMs: SCENARIO_TIMEOUT_MS + 30_000, + }, + ); + if (verbose) process.stderr.write(run.stderr); + + const checksPath = findFile(scenarioDir, "checks.json"); + if (!checksPath) { + record(result, "runner", "fail", `No checks.json (runner exit ${run.code}): ${run.stderr.trim().slice(-2000)}`); + return result; + } + const checks = JSON.parse(readFileSync(checksPath, "utf8")) as RawCheck[]; + for (const check of checks) { + const status = String(check.status ?? "FAILURE").toUpperCase(); + if (IGNORED_STATUSES.has(status)) continue; + const id = String(check.id ?? check.name ?? "unnamed"); + const message = String(check.errorMessage ?? check.description ?? ""); + record(result, id, status === "SUCCESS" ? "pass" : "fail", message); + } + // Whether pi completed the scenario. Some scenarios expect the client to give up, so this is + // baselined like any other check. + let client: { success?: unknown; error?: unknown } = { error: "client.ts wrote no report" }; + if (existsSync(reportPath)) client = JSON.parse(readFileSync(reportPath, "utf8")) as typeof client; + record(result, "pi-client", client.success === true ? "pass" : "fail", String(client.error ?? "")); + return result; +} + +function loadBaseline(): Baseline | undefined { + if (!existsSync(baselinePath)) return undefined; + return JSON.parse(readFileSync(baselinePath, "utf8")) as Baseline; +} + +function toBaseline(results: ScenarioResult[]): Baseline { + const baseline: Baseline = { conformance: CONFORMANCE_PACKAGE, modes: {} }; + for (const result of results) { + const ids = [...result.checks.keys()].sort(); + baseline.modes[result.mode] ??= {}; + baseline.modes[result.mode][result.scenario] = { + passing: ids.filter((id) => result.checks.get(id) === "pass"), + failing: ids.filter((id) => result.checks.get(id) === "fail"), + }; + } + return baseline; +} + +/** Compare with the baseline. Returns whether there is no regression. */ +function evaluate(results: ScenarioResult[], baseline: Baseline): boolean { + const regressions: string[] = []; + const fixed: string[] = []; + for (const result of results) { + const expected = baseline.modes[result.mode]?.[result.scenario]; + const label = `${result.mode} ${result.scenario}`; + for (const [id, status] of result.checks) { + const message = result.messages.get(id); + if (status === "fail" && !expected?.failing.includes(id)) { + regressions.push(`${label}: ${id} fails${message ? `: ${message}` : ""}`); + } + if (status === "pass" && expected?.failing.includes(id)) fixed.push(`${label}: ${id}`); + } + for (const id of expected?.passing ?? []) { + if (!result.checks.has(id)) regressions.push(`${label}: ${id} was not reported`); + } + } + for (const [mode, scenarios] of Object.entries(baseline.modes)) { + for (const scenario of Object.keys(scenarios)) { + if (!results.some((result) => result.mode === mode && result.scenario === scenario)) { + regressions.push(`${mode} ${scenario}: scenario did not run`); + } + } + } + if (fixed.length > 0) { + console.log( + `\nFixed since the baseline (run with --update-baseline):\n${fixed.map((line) => ` ${line}`).join("\n")}`, + ); + } + if (regressions.length > 0) { + console.log(`\nRegressions:\n${regressions.map((line) => ` ${line}`).join("\n")}`); + return false; + } + console.log("\nNo regressions against the baseline."); + return true; +} + +function printResults(results: ScenarioResult[]): void { + for (const mode of MODES) { + const modeResults = results.filter((result) => result.mode === mode); + if (modeResults.length === 0) continue; + console.log(`\n${mode}`); + for (const result of modeResults) { + const statuses = [...result.checks.values()]; + const failed = statuses.filter((status) => status === "fail").length; + const passed = statuses.length - failed; + console.log(` ${failed === 0 ? "pass" : "FAIL"} ${result.scenario} (${passed} passed, ${failed} failed)`); + for (const [id, status] of result.checks) { + if (status !== "fail") continue; + const message = result.messages.get(id)?.replace(/\s+/g, " ").slice(0, 300); + console.log(` ${id}${message ? `: ${message}` : ""}`); + } + } + } +} + +async function main(): Promise { + const { values } = parseArgs({ + options: { + mode: { type: "string", multiple: true }, + scenario: { type: "string", multiple: true }, + "update-baseline": { type: "boolean", default: false }, + "keep-results": { type: "boolean", default: false }, + verbose: { type: "boolean", default: false }, + help: { type: "boolean", short: "h", default: false }, + }, + }); + if (values.help) { + console.log(`Usage: node packages/coding-agent/test/mcp-conformance/run.ts [options] + +Options: + --mode Only run this protocol version (repeatable): ${MODES.join(", ")} + --scenario Only run this scenario (repeatable) + --update-baseline Write the results to baseline.json instead of comparing + --keep-results Keep the upstream checks.json and client output + --verbose Print the upstream runner output`); + return 0; + } + if (process.platform === "win32") { + console.error("The conformance runner needs a POSIX shell."); + return 1; + } + const modes = values.mode ?? [...MODES]; + for (const mode of modes) { + if (!(MODES as readonly string[]).includes(mode)) { + console.error(`Unsupported mode ${mode}. Supported: ${MODES.join(", ")}`); + return 1; + } + } + const partial = values.mode !== undefined || values.scenario !== undefined; + if (values["update-baseline"] && partial) { + console.error("--update-baseline needs a full run, without --mode or --scenario."); + return 1; + } + + const workDir = mkdtempSync(join(tmpdir(), "pi-mcp-conformance-")); + try { + // The upstream runner splits --command on spaces, so use a launcher path without any. + const launcher = join(workDir, "pi-client"); + writeFileSync( + launcher, + `#!/bin/sh\nexec ${JSON.stringify(process.execPath)} --import ${JSON.stringify(resolverUrl)} ${JSON.stringify(clientPath)} "$@"\n`, + ); + chmodSync(launcher, 0o755); + + const results: ScenarioResult[] = []; + for (const mode of modes) { + const scenarios = (await listScenarios(mode, workDir)).filter( + (scenario) => !values.scenario || values.scenario.includes(scenario), + ); + for (const scenario of scenarios) { + process.stderr.write(`${mode} ${scenario}\n`); + results.push(await runScenario(mode, scenario, launcher, workDir, values.verbose)); + } + } + if (results.length === 0) { + console.error("No scenarios matched."); + return 1; + } + printResults(results); + + if (values["update-baseline"]) { + writeFileSync(baselinePath, `${JSON.stringify(toBaseline(results), null, "\t")}\n`); + console.log(`\nWrote ${baselinePath}`); + return 0; + } + const baseline = loadBaseline(); + if (!baseline) { + console.error("\nNo baseline.json. Run with --update-baseline and review it."); + return 1; + } + if (baseline.conformance !== CONFORMANCE_PACKAGE) { + console.error(`\nbaseline.json is for ${baseline.conformance}, not ${CONFORMANCE_PACKAGE}. Regenerate it.`); + return 1; + } + const scoped: Baseline = partial + ? { + conformance: baseline.conformance, + modes: Object.fromEntries( + modes.map((mode) => [ + mode, + Object.fromEntries( + Object.entries(baseline.modes[mode] ?? {}).filter( + ([scenario]) => !values.scenario || values.scenario.includes(scenario), + ), + ), + ]), + ), + } + : baseline; + return evaluate(results, scoped) ? 0 : 1; + } finally { + if (values["keep-results"]) console.log(`\nResults kept in ${workDir}`); + else rmSync(workDir, { recursive: true, force: true }); + } +} + +process.exit(await main());