From b56a35f90fcdc99ccac02aeec5cdb1f8c69f307c Mon Sep 17 00:00:00 2001 From: Christian Klotz Date: Mon, 27 Jul 2026 17:40:14 +0300 Subject: [PATCH] fix(coding-agent): project typed eval outputs --- packages/evals/README.md | 5 +-- packages/evals/src/extensions.eval.ts | 32 ++++++++++++----- packages/evals/src/pi-harness.ts | 49 +++++++++++++-------------- 3 files changed, 49 insertions(+), 37 deletions(-) diff --git a/packages/evals/README.md b/packages/evals/README.md index dcc447b76..dabb0fa10 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -5,8 +5,9 @@ Behavioral evaluations for Pi using `vitest-evals`. `src/pi-harness.ts` adapts Pi's `AgentSession` directly to `vitest-evals`. Each run uses isolated temporary project and agent directories that are removed afterward. Judge harnesses are configured separately from the application harness. -Eval cases supply either one prompt or a JSON-safe sequence of prompt and reload steps. The harness executes that input, -normalizes the resulting session, and returns diagnostic artifacts without defining scenario-specific behavior. +Eval cases supply either one prompt or a JSON-safe sequence of prompt and reload steps. The harness executes that input +and normalizes the resulting session without defining scenario-specific behavior. Suites can configure a typed output +selector over the final response and `AgentSession` state. ## Running diff --git a/packages/evals/src/extensions.eval.ts b/packages/evals/src/extensions.eval.ts index cdd9cb5ff..af4f6924e 100644 --- a/packages/evals/src/extensions.eval.ts +++ b/packages/evals/src/extensions.eval.ts @@ -1,9 +1,25 @@ +import { stat } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; import { describeEval, toolCalls } from "vitest-evals"; import { createPiCodingAgentHarness } from "./pi-harness.ts"; -const piExtensionsHarness = createPiCodingAgentHarness({ name: "pi-coding-agent-extensions" }); +const piExtensionsHarness = createPiCodingAgentHarness({ + name: "pi-coding-agent-extensions", + output: async ({ response, session }) => { + const extensionPath = join(session.sessionManager.getCwd(), ".pi", "extensions", "hello.ts"); + const extensionCreated = await stat(extensionPath).then( + (file) => file.isFile(), + () => false, + ); + return { + response, + extensionCreated, + loadedExtensionCount: session.extensionRunner.getExtensionPaths().length, + helloToolLabel: session.getToolDefinition("hello")?.label ?? null, + }; + }, +}); describeEval("Pi extensions", { harness: piExtensionsHarness }, (it) => { it("creates, reloads, and uses a hello extension", async ({ run }) => { @@ -20,15 +36,13 @@ describeEval("Pi extensions", { harness: piExtensionsHarness }, (it) => { }, ]); - expect(result.output.trim()).toBe("Hello, Bob!"); + expect(result.output).toEqual({ + response: "Hello, Bob!", + extensionCreated: true, + loadedExtensionCount: 1, + helloToolLabel: "Hello", + }); expect(result.errors).toEqual([]); - expect(result.artifacts?.workspaceFiles).toContain(join(".pi", "extensions", "hello.ts")); - expect(result.artifacts?.reloads).toEqual([ - expect.objectContaining({ - loadedExtensionCount: 1, - activeTools: expect.arrayContaining([{ name: "hello", label: "Hello" }]), - }), - ]); expect(toolCalls(result.session)).toContainEqual({ name: "hello", arguments: { name: "Bob" }, diff --git a/packages/evals/src/pi-harness.ts b/packages/evals/src/pi-harness.ts index c4b59c4fa..ec230183f 100644 --- a/packages/evals/src/pi-harness.ts +++ b/packages/evals/src/pi-harness.ts @@ -1,7 +1,7 @@ import assert from "node:assert"; -import { mkdir, mkdtemp, readdir, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join, relative } from "node:path"; +import { join } from "node:path"; import { contentText } from "@earendil-works/pi-ai"; import { type AgentSession, @@ -14,6 +14,8 @@ import { } from "@earendil-works/pi-coding-agent"; import { createHarness, + type Harness, + type JsonValue, normalizeRecord, type SimpleHarnessResult, type TranscriptEvent, @@ -27,6 +29,10 @@ type PiCodingAgentHarnessOptions = { noTools?: CreateAgentSessionOptions["noTools"]; }; +type PiCodingAgentHarnessWithOutput = PiCodingAgentHarnessOptions & { + output: (args: { response: string; session: AgentSession }) => TOutput | Promise; +}; + function getRequiredModelSelection(): { provider: string; model: string } { const provider = process.env.PI_PROVIDER?.trim(); const model = process.env.PI_MODEL?.trim(); @@ -85,11 +91,11 @@ async function promptAgent(session: AgentSession, input: string, signal: AbortSi return output; } -async function runPiCodingAgent( +async function runPiCodingAgent( input: PiCodingAgentInput, signal: AbortSignal | undefined, - options: PiCodingAgentHarnessOptions, -): Promise> { + options: PiCodingAgentHarnessOptions | PiCodingAgentHarnessWithOutput, +): Promise> { signal?.throwIfAborted(); const selection = getRequiredModelSelection(); const modelRuntime = await ModelRuntime.create(); @@ -130,35 +136,20 @@ async function runPiCodingAgent( "Expected an isolated eval session to start without extensions", ); const steps = typeof input === "string" ? [{ type: "prompt" as const, content: input }] : input; - const reloads: Array<{ - loadedExtensionCount: number; - activeTools: Array<{ name: string; label: string }>; - }> = []; - let output: string | undefined; + let response: string | undefined; for (const step of steps) { if (step.type === "prompt") { - output = await promptAgent(evalSession, step.content, signal); + response = await promptAgent(evalSession, step.content, signal); } else { await evalSession.reload(); - reloads.push({ - loadedExtensionCount: evalSession.extensionRunner.getExtensionPaths().length, - activeTools: evalSession - .getActiveToolNames() - .map((name) => ({ name, label: evalSession.getToolDefinition(name)?.label ?? name })) - .sort((left, right) => left.name.localeCompare(right.name)), - }); } } - if (output === undefined) throw new Error("Pi eval input must include at least one prompt step."); - const workspaceFiles = (await readdir(cwd, { recursive: true, withFileTypes: true })) - .filter((entry) => entry.isFile()) - .map((entry) => relative(cwd, join(entry.parentPath, entry.name))) - .sort(); + if (response === undefined) throw new Error("Pi eval input must include at least one prompt step."); + const output = "output" in options ? await options.output({ response, session: evalSession }) : response; const stats = evalSession.getSessionStats(); return { output, events: toTranscriptEvents(evalSession.messages), - artifacts: { reloads, workspaceFiles }, usage: { provider: model.provider, model: model.id, @@ -180,8 +171,14 @@ async function runPiCodingAgent( } } -export function createPiCodingAgentHarness(options: PiCodingAgentHarnessOptions = {}) { - return createHarness({ +export function createPiCodingAgentHarness( + options: PiCodingAgentHarnessWithOutput, +): Harness; +export function createPiCodingAgentHarness(options?: PiCodingAgentHarnessOptions): Harness; +export function createPiCodingAgentHarness( + options: PiCodingAgentHarnessOptions | PiCodingAgentHarnessWithOutput = {}, +) { + return createHarness({ name: options.name ?? "pi-coding-agent", run: ({ input, signal }) => runPiCodingAgent(input, signal, options), });