From 6173017a7b5a9df4935a1da4c0f0b696afa0f6de Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sat, 25 Jul 2026 14:36:32 +0200 Subject: [PATCH] fix: drop harness-pi-ai and its legacy registry aliases from evals The harness declared peer deps on the pre-rename @mariozechner/pi-ai and @mariozechner/pi-agent-core names, so npm auto-installed our own code from the registry under an alias. sync-versions bumped those aliases in lockstep, which made every release fail because the new version is not published yet. Build the eval harness on createHarness from vitest-evals core instead. The pi-specific code was already ours; the harness only contributed agent-lifecycle plumbing and unused tool replay. --- package-lock.json | 106 ------------------------ packages/evals/README.md | 14 +--- packages/evals/package.json | 3 - packages/evals/src/pi-ai-compat.ts | 5 -- packages/evals/src/pi-harness.ts | 126 +++++++++++++---------------- packages/evals/vitest.config.ts | 14 +--- scripts/sync-versions.js | 20 +---- scripts/sync-versions.test.mjs | 4 +- 8 files changed, 66 insertions(+), 226 deletions(-) delete mode 100644 packages/evals/src/pi-ai-compat.ts diff --git a/package-lock.json b/package-lock.json index 10f4b3772..049b6ba8d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1483,52 +1483,6 @@ "node": ">= 10" } }, - "node_modules/@mariozechner/pi-agent-core": { - "name": "@earendil-works/pi-agent-core", - "version": "0.81.1", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.81.1.tgz", - "integrity": "sha512-yqbh68CyhqxMov/jUogFJfMqlu2Gd37GAki+tr59YCmAPHfomiCA5ESzusXtpGzABeiZFC/OrRdQ4GwCCOMIHA==", - "dev": true, - "license": "MIT", - "peer": true, - "dependencies": { - "@earendil-works/pi-ai": "^0.81.1", - "ignore": "7.0.5", - "typebox": "1.1.38", - "yaml": "2.9.0" - }, - "engines": { - "node": ">=22.19.0" - } - }, - "node_modules/@mariozechner/pi-ai": { - "name": "@earendil-works/pi-ai", - "version": "0.81.1", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.81.1.tgz", - "integrity": "sha512-hzHE7Z8l5mgJk+ke67Lge0rwS2+wbKJrFKl9o5M1R1rh33+cCT7D1AHz1OAtX5wFs90E1/BTGhyJRTUHaMxGvQ==", - "dev": true, - "license": "MIT", - "peer": true, - "dependencies": { - "@anthropic-ai/sdk": "0.91.1", - "@aws-sdk/client-bedrock-runtime": "3.1048.0", - "@google/genai": "1.52.0", - "@mistralai/mistralai": "2.2.6", - "@opentelemetry/api": "1.9.0", - "@smithy/node-http-handler": "4.7.3", - "http-proxy-agent": "7.0.2", - "https-proxy-agent": "7.0.6", - "openai": "6.26.0", - "partial-json": "0.1.7", - "typebox": "1.1.38" - }, - "bin": { - "pi-ai": "dist/cli.js" - }, - "engines": { - "node": ">=22.19.0" - } - }, "node_modules/@mistralai/mistralai": { "version": "2.2.6", "resolved": "https://registry.npmjs.org/@mistralai/mistralai/-/mistralai-2.2.6.tgz", @@ -2348,18 +2302,6 @@ "zod": ">=3 <5" } }, - "node_modules/@vitest-evals/harness-pi-ai": { - "version": "0.15.0", - "resolved": "https://registry.npmjs.org/@vitest-evals/harness-pi-ai/-/harness-pi-ai-0.15.0.tgz", - "integrity": "sha512-iigLAFAZdJ4wefndL2pkX69lJ682QisN8dNhXx/rsJQGTsjXA0dQekw5uww8CCwOIJVagyII8y9XOmdJutz73Q==", - "dev": true, - "license": "Apache-2.0", - "peerDependencies": { - "@mariozechner/pi-agent-core": ">=0.67 <1", - "@mariozechner/pi-ai": ">=0.67 <1", - "vitest-evals": "*" - } - }, "node_modules/@vitest-evals/report-ui": { "version": "0.15.0", "resolved": "https://registry.npmjs.org/@vitest-evals/report-ui/-/report-ui-0.15.0.tgz", @@ -5681,61 +5623,13 @@ "version": "0.82.0", "devDependencies": { "@earendil-works/pi-coding-agent": "^0.82.0", - "@mariozechner/pi-agent-core": "npm:@earendil-works/pi-agent-core@0.82.0", - "@mariozechner/pi-ai": "npm:@earendil-works/pi-ai@0.82.0", "@types/node": "24.12.4", - "@vitest-evals/harness-pi-ai": "0.15.0", "shx": "0.4.0", "typescript": "5.9.3", "vitest": "4.1.9", "vitest-evals": "0.15.0" } }, - "packages/evals/node_modules/@mariozechner/pi-agent-core": { - "name": "@earendil-works/pi-agent-core", - "version": "0.82.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-0.82.0.tgz", - "integrity": "sha512-bnS9DpOKK5T/F/gQkaOnYdMsuuciWiScfAHHWC+k5OQ0HxjSqMFQvp8keurULLoT4+v8NHv4V14pNvd4hsfC0Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@earendil-works/pi-ai": "^0.82.0", - "diff": "8.0.4", - "ignore": "7.0.5", - "typebox": "1.1.38", - "yaml": "2.9.0" - }, - "engines": { - "node": ">=22.19.0" - } - }, - "packages/evals/node_modules/@mariozechner/pi-ai": { - "name": "@earendil-works/pi-ai", - "version": "0.82.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-0.82.0.tgz", - "integrity": "sha512-8MvW9+zno13sXDuT2kFMnWeTNUufUhPeZDRVO+igGoBRCDWgn7Xh2FkRQI1mRuet6QhF4ENQuLYdIAOyG6BhNw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@anthropic-ai/sdk": "0.91.1", - "@aws-sdk/client-bedrock-runtime": "3.1048.0", - "@google/genai": "1.52.0", - "@mistralai/mistralai": "2.2.6", - "@opentelemetry/api": "1.9.0", - "@smithy/node-http-handler": "4.7.3", - "http-proxy-agent": "7.0.2", - "https-proxy-agent": "7.0.6", - "openai": "6.26.0", - "partial-json": "0.1.7", - "typebox": "1.1.38" - }, - "bin": { - "pi-ai": "dist/cli.js" - }, - "engines": { - "node": ">=22.19.0" - } - }, "packages/evals/node_modules/@types/node": { "version": "24.12.4", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.4.tgz", diff --git a/packages/evals/README.md b/packages/evals/README.md index 2e938cb0a..f4b005243 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -1,6 +1,9 @@ # Pi evals -Behavioral evaluations for Pi using `vitest-evals` and its Pi harness. +Behavioral evaluations for Pi using `vitest-evals`. + +The Pi harness in `src/pi-harness.ts` is built on `createHarness` from `vitest-evals/harness`. It drives a real +`createAgentSession` in a temporary workspace and reports the final assistant message, transcript, and token usage. ## Running @@ -23,12 +26,3 @@ npm run eval -- -t "capital of France" ``` Authentication is resolved by Pi's normal `ModelRuntime`. Subscription-backed providers such as `openai-codex` use credentials from the user's Pi configuration. API-backed providers use their standard environment variables, such as `OPENAI_API_KEY` for `openai`. - -## Pi harness compatibility - -`@vitest-evals/harness-pi-ai` 0.15.0 still declares and imports the former -`@mariozechner/pi-ai` and `@mariozechner/pi-agent-core` package names. The eval -package provides aliases at the current Pi version and routes the legacy AI -import through `src/pi-ai-compat.ts`. Keep those alias versions synchronized -with the Pi workspace version until the upstream harness supports the current -`@earendil-works` packages. diff --git a/packages/evals/package.json b/packages/evals/package.json index ab7773ffb..5b243e0bb 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -9,10 +9,7 @@ }, "devDependencies": { "@earendil-works/pi-coding-agent": "^0.82.0", - "@mariozechner/pi-agent-core": "npm:@earendil-works/pi-agent-core@0.82.0", - "@mariozechner/pi-ai": "npm:@earendil-works/pi-ai@0.82.0", "@types/node": "24.12.4", - "@vitest-evals/harness-pi-ai": "0.15.0", "shx": "0.4.0", "typescript": "5.9.3", "vitest-evals": "0.15.0", diff --git a/packages/evals/src/pi-ai-compat.ts b/packages/evals/src/pi-ai-compat.ts deleted file mode 100644 index 7e8f7ed31..000000000 --- a/packages/evals/src/pi-ai-compat.ts +++ /dev/null @@ -1,5 +0,0 @@ -// @vitest-evals/harness-pi-ai 0.15.0 imports completeSimple from the legacy -// @mariozechner/pi-ai entry point. Remove this shim when the harness supports -// the current @earendil-works/pi-ai API. -export * from "@earendil-works/pi-ai"; -export { completeSimple } from "@earendil-works/pi-ai/compat"; diff --git a/packages/evals/src/pi-harness.ts b/packages/evals/src/pi-harness.ts index 0585a13ea..dd09ee1e4 100644 --- a/packages/evals/src/pi-harness.ts +++ b/packages/evals/src/pi-harness.ts @@ -8,21 +8,10 @@ import { SessionManager, SettingsManager, } from "@earendil-works/pi-coding-agent"; -import { type PiAiCreateAgentArgs, type PiAiRuntime, type PiAiToolset, piAiHarness } from "@vitest-evals/harness-pi-ai"; +import { createHarness, type SimpleHarnessResult } from "vitest-evals/harness"; type EvalSession = Awaited>["session"]; -interface EvalRunResult { - output: string; - metrics: { - provider: string; - model: string; - inputTokens: number; - outputTokens: number; - totalTokens: number; - }; -} - async function cleanupEvalResources(session: EvalSession | undefined, removeRoot: () => Promise): Promise { const failures: unknown[] = []; if (session) { @@ -54,8 +43,7 @@ function getRequiredModelSelection(): { provider: string; model: string } { return { provider, model }; } -async function createPiEvalAgent({ context }: PiAiCreateAgentArgs) { - const signal = context.signal; +async function runPiEval(input: string, signal: AbortSignal | undefined): Promise> { signal?.throwIfAborted(); const selection = getRequiredModelSelection(); const modelRuntime = await ModelRuntime.create(); @@ -74,7 +62,7 @@ async function createPiEvalAgent({ context }: PiAiCreateAgentArgs) { const cwd = join(root, "workspace"); const agentDir = join(root, "agent"); const cleanup = () => rm(root, { recursive: true, force: true }); - let session: EvalSession | undefined; + let createdSession: EvalSession | undefined; try { signal?.throwIfAborted(); await Promise.all([mkdir(cwd), mkdir(agentDir)]); @@ -94,7 +82,7 @@ async function createPiEvalAgent({ context }: PiAiCreateAgentArgs) { await resourceLoader.reload(); signal?.throwIfAborted(); - session = ( + createdSession = ( await createAgentSession({ cwd, agentDir, @@ -110,73 +98,69 @@ async function createPiEvalAgent({ context }: PiAiCreateAgentArgs) { signal?.throwIfAborted(); } catch (error) { try { - await cleanupEvalResources(session, cleanup); + await cleanupEvalResources(createdSession, cleanup); } catch (cleanupError) { throw new AggregateError([error, cleanupError], "Pi eval setup failed and cleanup also failed."); } throw error; } - if (!session) { + if (!createdSession) { throw new Error("Pi eval session setup completed without a session."); } - return { - agent: session.agent, - async run(input: string, runtime: PiAiRuntime, string>) { - const abort = () => void session.abort(); - runtime.signal?.addEventListener("abort", abort, { once: true }); - let result: EvalRunResult; - try { - runtime.signal?.throwIfAborted(); - await session.prompt(input); - const assistant = session.messages - .slice() - .reverse() - .find((message) => message.role === "assistant"); - if (!assistant) { - throw new Error("Pi eval did not produce an assistant message."); - } - if (assistant.stopReason !== "stop") { - throw new Error(assistant.errorMessage ?? `Pi eval stopped unexpectedly: ${assistant.stopReason}`); - } + const session = createdSession; + const abort = () => void session.abort(); + signal?.addEventListener("abort", abort, { once: true }); + let result: SimpleHarnessResult; + try { + signal?.throwIfAborted(); + await session.prompt(input); + const assistant = session.messages + .slice() + .reverse() + .find((message) => message.role === "assistant"); + if (!assistant) { + throw new Error("Pi eval did not produce an assistant message."); + } + if (assistant.stopReason !== "stop") { + throw new Error(assistant.errorMessage ?? `Pi eval stopped unexpectedly: ${assistant.stopReason}`); + } - const output = session.getLastAssistantText(); - if (!output) { - throw new Error("Pi eval produced an empty assistant response."); - } + const output = session.getLastAssistantText(); + if (!output) { + throw new Error("Pi eval produced an empty assistant response."); + } - const stats = session.getSessionStats(); - runtime.events.assistant(output, { - provider: model.provider, - model: model.id, - }); - result = { - output, - metrics: { - provider: model.provider, - model: model.id, - inputTokens: stats.tokens.input, - outputTokens: stats.tokens.output, - totalTokens: stats.tokens.total, - }, - }; - } catch (error) { - runtime.signal?.removeEventListener("abort", abort); - try { - await cleanupEvalResources(session, cleanup); - } catch (cleanupError) { - throw new AggregateError([error, cleanupError], "Pi eval run failed and cleanup also failed."); - } - throw error; - } - runtime.signal?.removeEventListener("abort", abort); + const stats = session.getSessionStats(); + result = { + output, + events: [ + { type: "message", role: "user", content: input }, + { type: "message", role: "assistant", content: output }, + ], + usage: { + provider: model.provider, + model: model.id, + inputTokens: stats.tokens.input, + outputTokens: stats.tokens.output, + totalTokens: stats.tokens.total, + }, + }; + } catch (error) { + signal?.removeEventListener("abort", abort); + try { await cleanupEvalResources(session, cleanup); - return result; - }, - }; + } catch (cleanupError) { + throw new AggregateError([error, cleanupError], "Pi eval run failed and cleanup also failed."); + } + throw error; + } + signal?.removeEventListener("abort", abort); + await cleanupEvalResources(session, cleanup); + return result; } -export const piCodingAgentHarness = piAiHarness({ +export const piCodingAgentHarness = createHarness({ name: "pi-coding-agent", - agent: createPiEvalAgent, + run: ({ input, signal }) => runPiEval(input, signal), }); diff --git a/packages/evals/vitest.config.ts b/packages/evals/vitest.config.ts index 6e7c6aa1d..5dd4fc817 100644 --- a/packages/evals/vitest.config.ts +++ b/packages/evals/vitest.config.ts @@ -1,9 +1,6 @@ -import { fileURLToPath } from "node:url"; import { defineConfig, mergeConfig } from "vitest/config"; import baseConfig, { workspaceSourcePaths } from "../../vitest.base.ts"; -const piAiCompatIndex = fileURLToPath(new URL("./src/pi-ai-compat.ts", import.meta.url)); - export default mergeConfig( baseConfig, defineConfig({ @@ -14,18 +11,9 @@ export default mergeConfig( testTimeout: 120000, hookTimeout: 30000, reporters: ["vitest-evals/reporter"], - server: { - deps: { - inline: [/@vitest-evals\/harness-pi-ai/], - }, - }, }, resolve: { - alias: [ - { find: /^@earendil-works\/pi-coding-agent$/, replacement: workspaceSourcePaths.codingAgentIndex }, - { find: /^@mariozechner\/pi-ai$/, replacement: piAiCompatIndex }, - { find: /^@mariozechner\/pi-agent-core$/, replacement: workspaceSourcePaths.agentIndex }, - ], + alias: [{ find: /^@earendil-works\/pi-coding-agent$/, replacement: workspaceSourcePaths.codingAgentIndex }], }, }), ); diff --git a/scripts/sync-versions.js b/scripts/sync-versions.js index 01291d810..918112364 100644 --- a/scripts/sync-versions.js +++ b/scripts/sync-versions.js @@ -11,21 +11,6 @@ import { findPackageDirectories } from "./package-workspaces.mjs"; const GENERATED_PACKAGE_SUFFIXES = [join("coding-agent", "install-lock")]; -function synchronizedDependencyVersion(dependencyName, currentSpecifier, versionMap) { - const directVersion = versionMap.get(dependencyName); - if (directVersion) { - return `^${directVersion}`; - } - if (typeof currentSpecifier !== "string" || !currentSpecifier.startsWith("npm:")) { - return null; - } - - const separator = currentSpecifier.lastIndexOf("@"); - const aliasPackage = currentSpecifier.slice("npm:".length, separator); - const aliasVersion = versionMap.get(aliasPackage); - return aliasVersion ? `npm:${aliasPackage}@${aliasVersion}` : null; -} - const packageRoot = process.argv[2] ?? "packages"; const workspacePackages = findPackageDirectories(packageRoot) .filter((directory) => !GENERATED_PACKAGE_SUFFIXES.some((suffix) => directory.endsWith(suffix))) @@ -63,7 +48,10 @@ for (const pkg of workspacePackages) { } for (const [dependencyName, currentSpecifier] of Object.entries(dependencies)) { - const newSpecifier = synchronizedDependencyVersion(dependencyName, currentSpecifier, versionMap); + // Registry aliases such as `npm:@earendil-works/pi-ai@0.1.2` are never workspace-linked, + // so lockstep bumping them would point at a version that is not published yet. + const version = versionMap.get(dependencyName); + const newSpecifier = version ? `^${version}` : null; if (!newSpecifier || currentSpecifier === newSpecifier) { continue; } diff --git a/scripts/sync-versions.test.mjs b/scripts/sync-versions.test.mjs index 433ccb51c..173c18f0b 100644 --- a/scripts/sync-versions.test.mjs +++ b/scripts/sync-versions.test.mjs @@ -25,7 +25,7 @@ function runSyncVersions(root) { }); } -test("synchronizes private dependencies without touching generated manifests or relaxing published lockstep", async () => { +test("synchronizes private dependencies without touching registry aliases, generated manifests, or published lockstep", async () => { const root = await mkdtemp(join(tmpdir(), "pi-sync-versions-")); try { await writeManifest(root, "packages/ai", { @@ -59,7 +59,7 @@ test("synchronizes private dependencies without touching generated manifests or const evalsManifest = await readManifest(root, "packages/evals"); assert.equal(evalsManifest.dependencies["@earendil-works/pi-coding-agent"], "^2.0.0"); - assert.equal(evalsManifest.dependencies["@mariozechner/pi-ai"], "npm:@earendil-works/pi-ai@2.0.0"); + assert.equal(evalsManifest.dependencies["@mariozechner/pi-ai"], "npm:@earendil-works/pi-ai@1.0.0"); const generatedManifest = await readManifest(root, "packages/coding-agent/install-lock"); assert.equal(generatedManifest.dependencies["@earendil-works/pi-coding-agent"], "^1.0.0");