feat(coding-agent): add vitest eval harness (#7085)

package.json

@@ -26,10 +26,12 @@
 		"check:model-data": "npm --prefix packages/ai run check:model-data",
 		"generate:model-catalog": "npm --prefix packages/ai run generate-model-catalog",
 		"diff:model-catalog": "node scripts/diff-model-catalog.mjs",
+		"eval": "npm run eval --workspace=@earendil-works/pi-evals --",
 		"check:model-catalog": "node scripts/publish-model-catalog.mjs --input .artifacts/model-catalog --dry-run",
 		"profile:tui": "node scripts/profile-coding-agent-node.mjs --mode tui",
 		"profile:rpc": "node scripts/profile-coding-agent-node.mjs --mode rpc",
-		"test": "npm run test --workspaces --if-present",
+		"test": "npm run test:scripts && npm run test --workspaces --if-present",
+		"test:scripts": "node --test scripts/*.test.mjs",
 		"version:patch": "npm version patch -ws --no-git-tag-version && node scripts/sync-versions.js && npm install --package-lock-only --ignore-scripts",
 		"version:minor": "npm version minor -ws --no-git-tag-version && node scripts/sync-versions.js && npm install --package-lock-only --ignore-scripts",
 		"version:major": "npm version major -ws --no-git-tag-version && node scripts/sync-versions.js && npm install --package-lock-only --ignore-scripts",

packages/coding-agent/vitest.config.ts

@@ -1,38 +1,28 @@
-import { fileURLToPath } from "node:url";
-import { defineConfig } from "vitest/config";
+import { defineConfig, mergeConfig } from "vitest/config";
+import baseConfig, { workspaceSourcePaths } from "../../vitest.base.ts";
 
-const aiSrcIndex = fileURLToPath(new URL("../ai/src/index.ts", import.meta.url));
-const aiSrcCompat = fileURLToPath(new URL("../ai/src/compat.ts", import.meta.url));
-const aiSrcOAuth = fileURLToPath(new URL("../ai/src/oauth.ts", import.meta.url));
-const aiSrcProviders = fileURLToPath(new URL("../ai/src/providers", import.meta.url));
-const agentSrcIndex = fileURLToPath(new URL("../agent/src/index.ts", import.meta.url));
-const tuiSrcIndex = fileURLToPath(new URL("../tui/src/index.ts", import.meta.url));
-
-export default defineConfig({
-	test: {
-		globals: true,
-		environment: "node",
-		testTimeout: 30000,
-		reporters: process.env.GITHUB_ACTIONS ? ["dot", "github-actions"] : ["dot"],
-		silent: "passed-only",
-		server: {
-			deps: {
-				external: [/@silvia-odwyer\/photon-node/],
+export default mergeConfig(
+	baseConfig,
+	defineConfig({
+		test: {
+			globals: true,
+			environment: "node",
+			testTimeout: 30000,
+			reporters: process.env.GITHUB_ACTIONS ? ["dot", "github-actions"] : ["dot"],
+			silent: "passed-only",
+			server: {
+				deps: {
+					external: [/@silvia-odwyer\/photon-node/],
+				},
 			},
 		},
-	},
-	resolve: {
-		alias: [
-			{ find: /^@earendil-works\/pi-ai$/, replacement: aiSrcIndex },
-			{ find: /^@earendil-works\/pi-ai\/compat$/, replacement: aiSrcCompat },
-			{ find: /^@earendil-works\/pi-ai\/oauth$/, replacement: aiSrcOAuth },
-			{ find: /^@earendil-works\/pi-ai\/providers\/(.+)$/, replacement: `${aiSrcProviders}/$1.ts` },
-			{ find: /^@earendil-works\/pi-agent-core$/, replacement: agentSrcIndex },
-			{ find: /^@earendil-works\/pi-tui$/, replacement: tuiSrcIndex },
-			{ find: /^@mariozechner\/pi-ai$/, replacement: aiSrcIndex },
-			{ find: /^@mariozechner\/pi-ai\/oauth$/, replacement: aiSrcOAuth },
-			{ find: /^@mariozechner\/pi-agent-core$/, replacement: agentSrcIndex },
-			{ find: /^@mariozechner\/pi-tui$/, replacement: tuiSrcIndex },
-		],
-	},
-});
+		resolve: {
+			alias: [
+				{ find: /^@mariozechner\/pi-ai$/, replacement: workspaceSourcePaths.aiIndex },
+				{ find: /^@mariozechner\/pi-ai\/oauth$/, replacement: workspaceSourcePaths.aiOAuth },
+				{ find: /^@mariozechner\/pi-agent-core$/, replacement: workspaceSourcePaths.agentIndex },
+				{ find: /^@mariozechner\/pi-tui$/, replacement: workspaceSourcePaths.tuiIndex },
+			],
+		},
+	}),
+);

packages/evals/README.md

@@ -0,0 +1,34 @@
+# Pi evals
+
+Behavioral evaluations for Pi using `vitest-evals` and its Pi harness.
+
+## Running
+
+From the repository root, run with an explicit provider and model:
+
+```bash
+npm run eval -- --provider openai-codex --model gpt-5.4
+```
+
+When invoked from a Pi Bash tool, the current session supplies `PI_PROVIDER` and `PI_MODEL`, so this is sufficient:
+
+```bash
+npm run eval
+```
+
+The runner requires both values and never falls back to another model. Additional arguments are forwarded to Vitest, for example:
+
+```bash
+npm run eval -- -t "capital of France"
+```
+
+Authentication is resolved by Pi's normal `ModelRuntime`. Subscription-backed providers such as `openai-codex` use credentials from the user's Pi configuration. API-backed providers use their standard environment variables, such as `OPENAI_API_KEY` for `openai`.
+
+## Pi harness compatibility
+
+`@vitest-evals/harness-pi-ai` 0.15.0 still declares and imports the former
+`@mariozechner/pi-ai` and `@mariozechner/pi-agent-core` package names. The eval
+package provides aliases at the current Pi version and routes the legacy AI
+import through `src/pi-ai-compat.ts`. Keep those alias versions synchronized
+with the Pi workspace version until the upstream harness supports the current
+`@earendil-works` packages.

packages/evals/package.json

@@ -0,0 +1,21 @@
+{
+	"name": "@earendil-works/pi-evals",
+	"version": "0.82.0",
+	"private": true,
+	"type": "module",
+	"scripts": {
+		"clean": "shx rm -rf .eval",
+		"eval": "node scripts/run-evals.mjs"
+	},
+	"devDependencies": {
+		"@earendil-works/pi-coding-agent": "^0.82.0",
+		"@mariozechner/pi-agent-core": "npm:@earendil-works/pi-agent-core@0.82.0",
+		"@mariozechner/pi-ai": "npm:@earendil-works/pi-ai@0.82.0",
+		"@types/node": "24.12.4",
+		"@vitest-evals/harness-pi-ai": "0.15.0",
+		"shx": "0.4.0",
+		"typescript": "5.9.3",
+		"vitest-evals": "0.15.0",
+		"vitest": "4.1.9"
+	}
+}

packages/evals/scripts/run-evals.mjs

@@ -0,0 +1,78 @@
+import { spawnSync } from "node:child_process";
+import { createRequire } from "node:module";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const args = process.argv.slice(2);
+let provider;
+let model;
+let hasCliModelSelection = false;
+const vitestArgs = [];
+
+for (let index = 0; index < args.length; index += 1) {
+	const arg = args[index];
+	if (arg === "--provider" || arg === "--model") {
+		const value = args[index + 1];
+		if (!value || value.startsWith("-")) {
+			console.error(`Missing value for ${arg}`);
+			process.exit(1);
+		}
+		if (arg === "--provider") provider = value;
+		else model = value;
+		hasCliModelSelection = true;
+		index += 1;
+		continue;
+	}
+	if (arg.startsWith("--provider=")) {
+		provider = arg.slice("--provider=".length);
+		hasCliModelSelection = true;
+		continue;
+	}
+	if (arg.startsWith("--model=")) {
+		model = arg.slice("--model=".length);
+		hasCliModelSelection = true;
+		continue;
+	}
+	vitestArgs.push(arg);
+}
+
+if (hasCliModelSelection && (!provider || !model)) {
+	console.error("CLI model selection requires both --provider and --model.");
+	process.exit(1);
+}
+
+provider ??= process.env.PI_PROVIDER;
+model ??= process.env.PI_MODEL;
+
+if (!provider || !model) {
+	console.error(
+		"No eval model selected. Pass --provider and --model, or set PI_PROVIDER and PI_MODEL.",
+	);
+	process.exit(1);
+}
+
+const require = createRequire(import.meta.url);
+const vitestPackagePath = require.resolve("vitest/package.json");
+const vitestCliPath = resolve(dirname(vitestPackagePath), "vitest.mjs");
+
+console.error(`[eval] provider=${provider} model=${model}`);
+const result = spawnSync(
+	process.execPath,
+	[vitestCliPath, "run", "--config", "vitest.config.ts", ...vitestArgs],
+	{
+		cwd: packageRoot,
+		stdio: "inherit",
+		env: {
+			...process.env,
+			PI_PROVIDER: provider,
+			PI_MODEL: model,
+		},
+	},
+);
+
+if (result.error) {
+	throw result.error;
+}
+
+process.exit(result.status ?? 1);

packages/evals/src/general-knowledge.eval.ts

@@ -0,0 +1,15 @@
+import { expect } from "vitest";
+import { describeEval } from "vitest-evals";
+import { piCodingAgentHarness } from "./pi-harness.ts";
+
+describeEval("general knowledge", { harness: piCodingAgentHarness }, (it) => {
+	it("knows the capital of France", async ({ run }) => {
+		const result = await run("What's the capital of France? Respond with only the city name.");
+
+		expect(result.output.trim()).toBe("Paris");
+		expect(result.errors).toEqual([]);
+		expect(result.usage.provider).toBe(process.env.PI_PROVIDER);
+		expect(result.usage.model).toBe(process.env.PI_MODEL);
+		expect(result.usage.totalTokens).toBeGreaterThan(0);
+	});
+});

packages/evals/src/pi-ai-compat.ts

@@ -0,0 +1,5 @@
+// @vitest-evals/harness-pi-ai 0.15.0 imports completeSimple from the legacy
+// @mariozechner/pi-ai entry point. Remove this shim when the harness supports
+// the current @earendil-works/pi-ai API.
+export * from "@earendil-works/pi-ai";
+export { completeSimple } from "@earendil-works/pi-ai/compat";

packages/evals/src/pi-harness.ts

@@ -0,0 +1,182 @@
+import { mkdir, mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+	createAgentSession,
+	DefaultResourceLoader,
+	ModelRuntime,
+	SessionManager,
+	SettingsManager,
+} from "@earendil-works/pi-coding-agent";
+import { type PiAiCreateAgentArgs, type PiAiRuntime, type PiAiToolset, piAiHarness } from "@vitest-evals/harness-pi-ai";
+
+type EvalSession = Awaited<ReturnType<typeof createAgentSession>>["session"];
+
+interface EvalRunResult {
+	output: string;
+	metrics: {
+		provider: string;
+		model: string;
+		inputTokens: number;
+		outputTokens: number;
+		totalTokens: number;
+	};
+}
+
+async function cleanupEvalResources(session: EvalSession | undefined, removeRoot: () => Promise<void>): Promise<void> {
+	const failures: unknown[] = [];
+	if (session) {
+		try {
+			session.dispose();
+		} catch (error) {
+			failures.push(error);
+		}
+	}
+	try {
+		await removeRoot();
+	} catch (error) {
+		failures.push(error);
+	}
+	if (failures.length === 1) {
+		throw failures[0];
+	}
+	if (failures.length > 1) {
+		throw new AggregateError(failures, "Pi eval session disposal and temporary directory cleanup failed.");
+	}
+}
+
+function getRequiredModelSelection(): { provider: string; model: string } {
+	const provider = process.env.PI_PROVIDER?.trim();
+	const model = process.env.PI_MODEL?.trim();
+	if (!provider || !model) {
+		throw new Error("PI_PROVIDER and PI_MODEL must both be set for eval runs.");
+	}
+	return { provider, model };
+}
+
+async function createPiEvalAgent({ context }: PiAiCreateAgentArgs<string>) {
+	const signal = context.signal;
+	signal?.throwIfAborted();
+	const selection = getRequiredModelSelection();
+	const modelRuntime = await ModelRuntime.create();
+	signal?.throwIfAborted();
+	const model = modelRuntime.getModel(selection.provider, selection.model);
+	if (!model) {
+		throw new Error(`Eval model not found: ${selection.provider}/${selection.model}`);
+	}
+	const auth = await modelRuntime.checkAuth(selection.provider);
+	signal?.throwIfAborted();
+	if (!auth) {
+		throw new Error(`No authentication configured for eval model ${selection.provider}/${selection.model}.`);
+	}
+
+	const root = await mkdtemp(join(tmpdir(), "pi-eval-"));
+	const cwd = join(root, "workspace");
+	const agentDir = join(root, "agent");
+	const cleanup = () => rm(root, { recursive: true, force: true });
+	let session: EvalSession | undefined;
+	try {
+		signal?.throwIfAborted();
+		await Promise.all([mkdir(cwd), mkdir(agentDir)]);
+		signal?.throwIfAborted();
+
+		const settingsManager = SettingsManager.inMemory();
+		const resourceLoader = new DefaultResourceLoader({
+			cwd,
+			agentDir,
+			settingsManager,
+			noExtensions: true,
+			noSkills: true,
+			noPromptTemplates: true,
+			noThemes: true,
+			noContextFiles: true,
+		});
+		await resourceLoader.reload();
+		signal?.throwIfAborted();
+
+		session = (
+			await createAgentSession({
+				cwd,
+				agentDir,
+				model,
+				modelRuntime,
+				thinkingLevel: "off",
+				noTools: "all",
+				resourceLoader,
+				settingsManager,
+				sessionManager: SessionManager.inMemory(cwd),
+			})
+		).session;
+		signal?.throwIfAborted();
+	} catch (error) {
+		try {
+			await cleanupEvalResources(session, cleanup);
+		} catch (cleanupError) {
+			throw new AggregateError([error, cleanupError], "Pi eval setup failed and cleanup also failed.");
+		}
+		throw error;
+	}
+	if (!session) {
+		throw new Error("Pi eval session setup completed without a session.");
+	}
+
+	return {
+		agent: session.agent,
+		async run(input: string, runtime: PiAiRuntime<PiAiToolset<string>, string>) {
+			const abort = () => void session.abort();
+			runtime.signal?.addEventListener("abort", abort, { once: true });
+			let result: EvalRunResult;
+			try {
+				runtime.signal?.throwIfAborted();
+				await session.prompt(input);
+				const assistant = session.messages
+					.slice()
+					.reverse()
+					.find((message) => message.role === "assistant");
+				if (!assistant) {
+					throw new Error("Pi eval did not produce an assistant message.");
+				}
+				if (assistant.stopReason !== "stop") {
+					throw new Error(assistant.errorMessage ?? `Pi eval stopped unexpectedly: ${assistant.stopReason}`);
+				}
+
+				const output = session.getLastAssistantText();
+				if (!output) {
+					throw new Error("Pi eval produced an empty assistant response.");
+				}
+
+				const stats = session.getSessionStats();
+				runtime.events.assistant(output, {
+					provider: model.provider,
+					model: model.id,
+				});
+				result = {
+					output,
+					metrics: {
+						provider: model.provider,
+						model: model.id,
+						inputTokens: stats.tokens.input,
+						outputTokens: stats.tokens.output,
+						totalTokens: stats.tokens.total,
+					},
+				};
+			} catch (error) {
+				runtime.signal?.removeEventListener("abort", abort);
+				try {
+					await cleanupEvalResources(session, cleanup);
+				} catch (cleanupError) {
+					throw new AggregateError([error, cleanupError], "Pi eval run failed and cleanup also failed.");
+				}
+				throw error;
+			}
+			runtime.signal?.removeEventListener("abort", abort);
+			await cleanupEvalResources(session, cleanup);
+			return result;
+		},
+	};
+}
+
+export const piCodingAgentHarness = piAiHarness({
+	name: "pi-coding-agent",
+	agent: createPiEvalAgent,
+});

packages/evals/tsconfig.json

@@ -0,0 +1,10 @@
+{
+	"extends": "../../tsconfig.json",
+	"compilerOptions": {
+		"noEmit": true,
+		"module": "NodeNext",
+		"moduleResolution": "NodeNext",
+		"types": ["node", "vitest"]
+	},
+	"include": ["src/**/*.ts"]
+}

packages/evals/vitest.config.ts

@@ -0,0 +1,31 @@
+import { fileURLToPath } from "node:url";
+import { defineConfig, mergeConfig } from "vitest/config";
+import baseConfig, { workspaceSourcePaths } from "../../vitest.base.ts";
+
+const piAiCompatIndex = fileURLToPath(new URL("./src/pi-ai-compat.ts", import.meta.url));
+
+export default mergeConfig(
+	baseConfig,
+	defineConfig({
+		test: {
+			environment: "node",
+			fileParallelism: false,
+			include: ["src/**/*.eval.ts"],
+			testTimeout: 120000,
+			hookTimeout: 30000,
+			reporters: ["vitest-evals/reporter"],
+			server: {
+				deps: {
+					inline: [/@vitest-evals\/harness-pi-ai/],
+				},
+			},
+		},
+		resolve: {
+			alias: [
+				{ find: /^@earendil-works\/pi-coding-agent$/, replacement: workspaceSourcePaths.codingAgentIndex },
+				{ find: /^@mariozechner\/pi-ai$/, replacement: piAiCompatIndex },
+				{ find: /^@mariozechner\/pi-agent-core$/, replacement: workspaceSourcePaths.agentIndex },
+			],
+		},
+	}),
+);

scripts/sync-versions.js

@@ -1,29 +1,47 @@
 #!/usr/bin/env node
 
 /**
- * Syncs all non-private workspace package dependency versions to match their current versions.
- * This ensures release packages, including unpublished packages, use lockstep versioning.
+ * Validates lockstep versions for published packages, then synchronizes
+ * internal dependency versions in all workspace packages, including private ones.
  */
 
 import { readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { findPackageDirectories } from "./package-workspaces.mjs";
 
-const packages = findPackageDirectories()
+const GENERATED_PACKAGE_SUFFIXES = [join("coding-agent", "install-lock")];
+
+function synchronizedDependencyVersion(dependencyName, currentSpecifier, versionMap) {
+	const directVersion = versionMap.get(dependencyName);
+	if (directVersion) {
+		return `^${directVersion}`;
+	}
+	if (typeof currentSpecifier !== "string" || !currentSpecifier.startsWith("npm:")) {
+		return null;
+	}
+
+	const separator = currentSpecifier.lastIndexOf("@");
+	const aliasPackage = currentSpecifier.slice("npm:".length, separator);
+	const aliasVersion = versionMap.get(aliasPackage);
+	return aliasVersion ? `npm:${aliasPackage}@${aliasVersion}` : null;
+}
+
+const packageRoot = process.argv[2] ?? "packages";
+const workspacePackages = findPackageDirectories(packageRoot)
+	.filter((directory) => !GENERATED_PACKAGE_SUFFIXES.some((suffix) => directory.endsWith(suffix)))
 	.map((directory) => {
 		const path = join(directory, "package.json");
 		return { data: JSON.parse(readFileSync(path, "utf8")), path };
-	})
-	.filter((pkg) => pkg.data.private !== true);
-
-const versionMap = new Map(packages.map((pkg) => [pkg.data.name, pkg.data.version]));
+	});
+const publishedPackages = workspacePackages.filter((pkg) => pkg.data.private !== true);
+const versionMap = new Map(workspacePackages.map((pkg) => [pkg.data.name, pkg.data.version]));
 
 console.log("Current versions:");
-for (const [name, version] of [...versionMap].sort(([a], [b]) => a.localeCompare(b))) {
-	console.log(`  ${name}: ${version}`);
+for (const pkg of [...publishedPackages].sort((a, b) => a.data.name.localeCompare(b.data.name))) {
+	console.log(`  ${pkg.data.name}: ${pkg.data.version}`);
 }
 
-const versions = new Set(versionMap.values());
+const versions = new Set(publishedPackages.map((pkg) => pkg.data.version));
 if (versions.size > 1) {
 	console.error("\nERROR: Not all non-private packages have the same version.");
 	console.error("Expected lockstep versioning. Run one of:");
@@ -36,37 +54,33 @@ if (versions.size > 1) {
 console.log("\nAll non-private packages are at the same version (lockstep).");
 
 let totalUpdates = 0;
-for (const pkg of packages) {
-	let updated = false;
-
+const updatedPackages = new Set();
+for (const pkg of workspacePackages) {
 	for (const dependencyType of ["dependencies", "devDependencies"]) {
 		const dependencies = pkg.data[dependencyType];
 		if (!dependencies) {
 			continue;
 		}
 
-		for (const [dependencyName, currentVersion] of Object.entries(dependencies)) {
-			const dependencyVersion = versionMap.get(dependencyName);
-			if (!dependencyVersion) {
-				continue;
-			}
-
-			const newVersion = `^${dependencyVersion}`;
-			if (currentVersion === newVersion) {
+		for (const [dependencyName, currentSpecifier] of Object.entries(dependencies)) {
+			const newSpecifier = synchronizedDependencyVersion(dependencyName, currentSpecifier, versionMap);
+			if (!newSpecifier || currentSpecifier === newSpecifier) {
 				continue;
 			}
 
 			console.log(`\n${pkg.data.name}:`);
-			console.log(`  ${dependencyName}: ${currentVersion} → ${newVersion}${dependencyType === "devDependencies" ? " (devDependencies)" : ""}`);
-			dependencies[dependencyName] = newVersion;
-			updated = true;
+			console.log(
+				`  ${dependencyName}: ${currentSpecifier} → ${newSpecifier}${dependencyType === "devDependencies" ? " (devDependencies)" : ""}`,
+			);
+			dependencies[dependencyName] = newSpecifier;
+			updatedPackages.add(pkg);
 			totalUpdates++;
 		}
 	}
+}
 
-	if (updated) {
-		writeFileSync(pkg.path, `${JSON.stringify(pkg.data, null, "\t")}\n`);
-	}
+for (const pkg of updatedPackages) {
+	writeFileSync(pkg.path, `${JSON.stringify(pkg.data, null, "\t")}\n`);
 }
 
 if (totalUpdates === 0) {

scripts/sync-versions.test.mjs

@@ -0,0 +1,75 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import test from "node:test";
+
+const syncVersionsScript = fileURLToPath(new URL("./sync-versions.js", import.meta.url));
+
+async function writeManifest(root, relativeDirectory, manifest) {
+	const directory = join(root, relativeDirectory);
+	await mkdir(directory, { recursive: true });
+	await writeFile(join(directory, "package.json"), `${JSON.stringify(manifest, null, "\t")}\n`);
+}
+
+async function readManifest(root, relativeDirectory) {
+	return JSON.parse(await readFile(join(root, relativeDirectory, "package.json"), "utf8"));
+}
+
+function runSyncVersions(root) {
+	return spawnSync(process.execPath, [syncVersionsScript, join(root, "packages")], {
+		cwd: root,
+		encoding: "utf8",
+	});
+}
+
+test("synchronizes private dependencies without touching generated manifests or relaxing published lockstep", async () => {
+	const root = await mkdtemp(join(tmpdir(), "pi-sync-versions-"));
+	try {
+		await writeManifest(root, "packages/ai", {
+			name: "@earendil-works/pi-ai",
+			version: "2.0.0",
+		});
+		await writeManifest(root, "packages/coding-agent", {
+			name: "@earendil-works/pi-coding-agent",
+			version: "2.0.0",
+		});
+		await writeManifest(root, "packages/evals", {
+			name: "@earendil-works/pi-evals",
+			version: "9.9.9",
+			private: true,
+			dependencies: {
+				"@earendil-works/pi-coding-agent": "^1.0.0",
+				"@mariozechner/pi-ai": "npm:@earendil-works/pi-ai@1.0.0",
+			},
+		});
+		await writeManifest(root, "packages/coding-agent/install-lock", {
+			name: "generated-install-lock",
+			version: "0.0.0",
+			private: true,
+			dependencies: {
+				"@earendil-works/pi-coding-agent": "^1.0.0",
+			},
+		});
+
+		const result = runSyncVersions(root);
+		assert.equal(result.status, 0, result.stderr);
+
+		const evalsManifest = await readManifest(root, "packages/evals");
+		assert.equal(evalsManifest.dependencies["@earendil-works/pi-coding-agent"], "^2.0.0");
+		assert.equal(evalsManifest.dependencies["@mariozechner/pi-ai"], "npm:@earendil-works/pi-ai@2.0.0");
+		const generatedManifest = await readManifest(root, "packages/coding-agent/install-lock");
+		assert.equal(generatedManifest.dependencies["@earendil-works/pi-coding-agent"], "^1.0.0");
+
+		await writeManifest(root, "packages/ai", {
+			name: "@earendil-works/pi-ai",
+			version: "3.0.0",
+		});
+		const lockstepFailure = runSyncVersions(root);
+		assert.equal(lockstepFailure.status, 1, lockstepFailure.stderr);
+	} finally {
+		await rm(root, { recursive: true, force: true });
+	}
+});

vitest.base.ts

@@ -0,0 +1,28 @@
+import { fileURLToPath } from "node:url";
+import { defineConfig } from "vitest/config";
+
+export const workspaceSourcePaths = {
+	aiIndex: fileURLToPath(new URL("./packages/ai/src/index.ts", import.meta.url)),
+	aiCompat: fileURLToPath(new URL("./packages/ai/src/compat.ts", import.meta.url)),
+	aiOAuth: fileURLToPath(new URL("./packages/ai/src/oauth.ts", import.meta.url)),
+	aiProviders: fileURLToPath(new URL("./packages/ai/src/providers", import.meta.url)),
+	agentIndex: fileURLToPath(new URL("./packages/agent/src/index.ts", import.meta.url)),
+	codingAgentIndex: fileURLToPath(new URL("./packages/coding-agent/src/index.ts", import.meta.url)),
+	tuiIndex: fileURLToPath(new URL("./packages/tui/src/index.ts", import.meta.url)),
+} as const;
+
+export default defineConfig({
+	resolve: {
+		alias: [
+			{ find: /^@earendil-works\/pi-ai$/, replacement: workspaceSourcePaths.aiIndex },
+			{ find: /^@earendil-works\/pi-ai\/compat$/, replacement: workspaceSourcePaths.aiCompat },
+			{ find: /^@earendil-works\/pi-ai\/oauth$/, replacement: workspaceSourcePaths.aiOAuth },
+			{
+				find: /^@earendil-works\/pi-ai\/providers\/(.+)$/,
+				replacement: `${workspaceSourcePaths.aiProviders}/$1.ts`,
+			},
+			{ find: /^@earendil-works\/pi-agent-core$/, replacement: workspaceSourcePaths.agentIndex },
+			{ find: /^@earendil-works\/pi-tui$/, replacement: workspaceSourcePaths.tuiIndex },
+		],
+	},
+});