feat(coding-agent): add extension creation eval

biome.json

@@ -34,6 +34,7 @@
 			"!**/test-sessions.ts",
 			"!**/models.generated.ts",
 			"!**/*.models.ts",
+			"!packages/evals/src/pi-harness.ts",
 			"!packages/mom/data/**/*",
 			"!!**/node_modules"
 		]

package-lock.json

@@ -5622,6 +5622,7 @@
 			"name": "@earendil-works/pi-evals",
 			"version": "0.82.1",
 			"devDependencies": {
+				"@earendil-works/pi-ai": "^0.82.1",
 				"@earendil-works/pi-coding-agent": "^0.82.1",
 				"@types/node": "24.12.4",
 				"shx": "0.4.0",

packages/evals/README.md

@@ -2,8 +2,13 @@
 
 Behavioral evaluations for Pi using `vitest-evals`.
 
-The Pi harness in `src/pi-harness.ts` is built on `createHarness` from `vitest-evals/harness`. It drives a real
-`createAgentSession` in a temporary workspace and reports the final assistant message, transcript, and token usage.
+`src/pi-harness.ts` mirrors the upstream `@vitest-evals/harness-pi-ai` implementation, except that it imports Pi AI
+from `@earendil-works/pi-ai/compat`. Keep it synchronized with
+[`getsentry/vitest-evals`](https://github.com/getsentry/vitest-evals/blob/main/packages/harness-pi-ai/src/index.ts).
+
+Pi's `AgentSession` is the system under test. `src/pi-agent.ts` adapts it to the harness's agent contract, while
+`piAiHarness(...)` handles eval execution and trace normalization. Each run uses a temporary workspace that is removed
+afterward.
 
 ## Running
 

packages/evals/package.json

@@ -9,6 +9,7 @@
 	},
 	"devDependencies": {
 		"@earendil-works/pi-coding-agent": "^0.82.1",
+		"@earendil-works/pi-ai": "^0.82.1",
 		"@types/node": "24.12.4",
 		"shx": "0.4.0",
 		"typescript": "5.9.3",

packages/evals/src/extensions.eval.ts

@@ -0,0 +1,49 @@
+import { stat } from "node:fs/promises";
+import { join } from "node:path";
+import { expect } from "vitest";
+import { describeEval } from "vitest-evals";
+import { createPiEvalAgent } from "./pi-agent.ts";
+import { piAiHarness } from "./pi-harness.ts";
+
+const createHelloExtensionPrompt =
+	"Create a Pi extension with a hello tool that takes a name and returns a greeting. For example, passing Bob should return `Hello, Bob!`.";
+
+const piExtensionsHarness = piAiHarness({
+	name: "pi-coding-agent-extensions",
+	agent: ({ context }) =>
+		createPiEvalAgent(
+			{
+				run: async ({ input, cwd, prompt, session }) => {
+					expect(session.extensionRunner.getExtensionPaths()).toHaveLength(0);
+					await prompt(input);
+					const extensionPath = join(cwd, ".pi", "extensions", "hello.ts");
+					const extensionCreated = await stat(extensionPath).then(
+						(file) => file.isFile(),
+						() => false,
+					);
+					if (!extensionCreated) throw new Error(`Extension was not created at ${extensionPath}`);
+
+					await session.reload();
+					expect(session.extensionRunner.getExtensionPaths()).toHaveLength(1);
+					if (!session.getActiveToolNames().includes("hello")) {
+						throw new Error("Extension tool 'hello' was not loaded");
+					}
+					if (session.getToolDefinition("hello")?.label !== "Hello") {
+						throw new Error("Extension tool 'hello' did not have label 'Hello'");
+					}
+
+					return prompt("Use the hello tool to greet Bob. Respond with exactly the text returned by the tool.");
+				},
+			},
+			context.signal,
+		),
+});
+
+describeEval("Pi extensions", { harness: piExtensionsHarness }, (it) => {
+	it("creates, reloads, and uses a hello extension", async ({ run }) => {
+		const result = await run(createHelloExtensionPrompt);
+
+		expect(result.output.trim()).toBe("Hello, Bob!");
+		expect(result.errors).toEqual([]);
+	});
+});

packages/evals/src/pi-agent.ts

@@ -0,0 +1,137 @@
+import { mkdir, mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+	createAgentSessionFromServices,
+	createAgentSessionServices,
+	ModelRuntime,
+	SessionManager,
+	SettingsManager,
+} from "@earendil-works/pi-coding-agent";
+import type { PiAiEventSink, PiAiRuntime, PiAiToolset } from "./pi-harness.ts";
+
+type EvalSession = Awaited<ReturnType<typeof createAgentSessionFromServices>>["session"];
+type EvalRuntime = PiAiRuntime<PiAiToolset<string>, string>;
+
+type PiEvalAgentOptions = {
+	noTools?: "all" | "builtin";
+	run?: (args: {
+		input: string;
+		cwd: string;
+		session: EvalSession;
+		prompt: (input: string) => Promise<string>;
+	}) => Promise<string>;
+};
+
+function getRequiredModelSelection(): { provider: string; model: string } {
+	const provider = process.env.PI_PROVIDER?.trim();
+	const model = process.env.PI_MODEL?.trim();
+	if (!provider || !model) {
+		throw new Error("PI_PROVIDER and PI_MODEL must both be set for eval runs.");
+	}
+	return { provider, model };
+}
+
+async function promptPi(
+	session: EvalSession,
+	input: string,
+	signal: AbortSignal | undefined,
+	events: PiAiEventSink,
+	recordUser: boolean,
+): Promise<string> {
+	signal?.throwIfAborted();
+	if (recordUser) events.user(input);
+	await session.prompt(input);
+
+	const assistant = session.messages
+		.slice()
+		.reverse()
+		.find((message) => message.role === "assistant");
+	if (!assistant) throw new Error("Pi eval did not produce an assistant message.");
+	if (assistant.stopReason !== "stop") {
+		throw new Error(assistant.errorMessage ?? `Pi eval stopped unexpectedly: ${assistant.stopReason}`);
+	}
+
+	const output = session.getLastAssistantText();
+	if (!output) throw new Error("Pi eval produced an empty assistant response.");
+	events.assistant(output);
+	return output;
+}
+
+export async function createPiEvalAgent(options: PiEvalAgentOptions, signal: AbortSignal | undefined) {
+	signal?.throwIfAborted();
+	const selection = getRequiredModelSelection();
+	const modelRuntime = await ModelRuntime.create();
+	const model = modelRuntime.getModel(selection.provider, selection.model);
+	if (!model) throw new Error(`Eval model not found: ${selection.provider}/${selection.model}`);
+
+	const root = await mkdtemp(join(tmpdir(), "pi-eval-"));
+	const cwd = join(root, "workspace");
+	const agentDir = join(root, "agent");
+	let session: EvalSession | undefined;
+	const cleanup = async () => {
+		try {
+			session?.dispose();
+		} finally {
+			await rm(root, { recursive: true, force: true });
+		}
+	};
+
+	try {
+		await Promise.all([mkdir(cwd), mkdir(agentDir)]);
+		const services = await createAgentSessionServices({
+			cwd,
+			agentDir,
+			modelRuntime,
+			settingsManager: SettingsManager.inMemory(),
+		});
+		signal?.throwIfAborted();
+		session = (
+			await createAgentSessionFromServices({
+				services,
+				sessionManager: SessionManager.inMemory(cwd),
+				model,
+				thinkingLevel: "off",
+				noTools: options.noTools,
+			})
+		).session;
+	} catch (error) {
+		await cleanup();
+		throw error;
+	}
+
+	const evalSession = session;
+	return {
+		agent: evalSession.agent,
+		async run(input: string, runtime: EvalRuntime) {
+			const abort = () => void evalSession.abort();
+			let promptCount = 0;
+			runtime.signal?.addEventListener("abort", abort, { once: true });
+			try {
+				const prompt = (promptInput: string) => {
+					const recordUser = promptCount > 0;
+					promptCount++;
+					return promptPi(evalSession, promptInput, runtime.signal, runtime.events, recordUser);
+				};
+				const output = options.run
+					? await options.run({ input, cwd, session: evalSession, prompt })
+					: await prompt(input);
+				const stats = evalSession.getSessionStats();
+				return {
+					output,
+					metrics: {
+						provider: model.provider,
+						model: model.id,
+						inputTokens: stats.tokens.input,
+						outputTokens: stats.tokens.output,
+						totalTokens: stats.tokens.total,
+						toolCalls: stats.toolCalls,
+					},
+				};
+			} finally {
+				runtime.signal?.removeEventListener("abort", abort);
+				await cleanup();
+			}
+		},
+	};
+}

packages/evals/src/smoke.eval.ts

@@ -1,9 +1,14 @@
 import { expect } from "vitest";
 import { describeEval } from "vitest-evals";
-import { piCodingAgentHarness } from "./pi-harness.ts";
+import { createPiEvalAgent } from "./pi-agent.ts";
+import { piAiHarness } from "./pi-harness.ts";
 
-describeEval("general knowledge", { harness: piCodingAgentHarness }, (it) => {
-	it("knows the capital of France", async ({ run }) => {
+const piCodingAgentHarness = piAiHarness({
+	agent: ({ context }) => createPiEvalAgent({ noTools: "all" }, context.signal),
+});
+
+describeEval("Pi Coding Agent smoke", { harness: piCodingAgentHarness }, (it) => {
+	it("runs a basic prompt end to end", async ({ run }) => {
 		const result = await run("What's the capital of France? Respond with only the city name.");
 
 		expect(result.output.trim()).toBe("Paris");