fix(coding-agent): validate eval prompts from transcripts (#9706)

Preserve partial-run usage and session artifacts when post-run validation fails.

packages/evals/src/harness.ts

@@ -6,6 +6,7 @@ import { tmpdir } from "node:os";
 import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { performance } from "node:perf_hooks";
 import { contentText, InMemoryCredentialStore } from "@earendil-works/pi-ai";
+import { getCurrentSystemPrompt } from "@earendil-works/pi-ai/utils/transcript";
 import {
 	type AgentSession,
 	type CreateAgentSessionOptions,
@@ -18,18 +19,27 @@ import {
 	SessionManager,
 } from "@earendil-works/pi-coding-agent";
 import {
+	attachHarnessRunToError,
 	createHarness,
 	type Harness,
 	type HarnessContext,
 	type JsonValue,
+	normalizeHarnessRun,
 	normalizeRecord,
 	type SimpleHarnessResult,
 	type TranscriptEvent,
 	toJsonValue,
+	type UsageSummary,
 } from "vitest-evals/harness";
 import type { DocumentationVariant } from "./plan.ts";
 import { PI_SESSION_SNAPSHOT_ARTIFACT } from "./report.ts";
 
+type PiRunDiagnostics = {
+	events: TranscriptEvent[];
+	metadata: Record<string, unknown>;
+	usage: UsageSummary;
+};
+
 export type PiCodingAgentInput = string | Array<{ type: "prompt"; content: string } | { type: "reload" }>;
 
 export type PiCodingAgentModelSelection = {
@@ -244,8 +254,12 @@ async function promptAgent(session: AgentSession, input: string, signal: AbortSi
 	return output ?? "";
 }
 
-function verifySystemPrompt(systemPrompt: string, options: PiCodingAgentHarnessOptions): void {
-	if (options.expectedPiDocumentation === undefined) return;
+export function verifySystemPrompt(
+	messages: AgentSession["messages"],
+	options: Pick<PiCodingAgentHarnessOptions, "name" | "expectedPiDocumentation">,
+): string {
+	const systemPrompt = getCurrentSystemPrompt(messages);
+	if (options.expectedPiDocumentation === undefined) return systemPrompt;
 	if (!systemPrompt.includes("\n<rules>\n")) {
 		throw new Error(`Pi system prompt lost its rules in the ${options.name} eval variant.`);
 	}
@@ -253,6 +267,7 @@ function verifySystemPrompt(systemPrompt: string, options: PiCodingAgentHarnessO
 	if (hasDocumentation !== options.expectedPiDocumentation) {
 		throw new Error(`Pi system prompt does not match the ${options.name} eval variant.`);
 	}
+	return systemPrompt;
 }
 
 async function runPiCodingAgent<TOutput extends JsonValue>(
@@ -285,6 +300,7 @@ async function runPiCodingAgent<TOutput extends JsonValue>(
 	let sessionManager: SessionManager | undefined;
 	let session: AgentSession | undefined;
 	let result: SimpleHarnessResult<string | TOutput> | undefined;
+	let runDiagnostics: PiRunDiagnostics | undefined;
 	let runError: unknown;
 	const cleanupErrors: unknown[] = [];
 	let hiddenCredentialEnvironment: { name: string; value: string } | undefined;
@@ -345,7 +361,6 @@ async function runPiCodingAgent<TOutput extends JsonValue>(
 
 		await enterToolSandbox(root, sandboxIdentity);
 		let response: string | undefined;
-		let systemPrompt: string | undefined;
 		const steps = typeof input === "string" ? [{ type: "prompt" as const, content: input }] : input;
 		let abortPromise: Promise<void> | undefined;
 		const abort = () => {
@@ -359,25 +374,21 @@ async function runPiCodingAgent<TOutput extends JsonValue>(
 					continue;
 				}
 				response = await promptAgent(session, step.content, signal);
-				systemPrompt = session.systemPrompt;
 			}
 		} finally {
 			signal?.removeEventListener("abort", abort);
 			if (abortPromise) await abortPromise;
 		}
-		if (response === undefined || systemPrompt === undefined) {
+		if (response === undefined) {
 			throw new Error("Pi eval input must include at least one prompt step.");
 		}
-		verifySystemPrompt(systemPrompt, options);
-		const output =
-			"output" in options ? await options.output({ response, session, systemPrompt, agentDir }) : response;
+		const systemPrompt = getCurrentSystemPrompt(session.messages);
 		const stats = session.getSessionStats();
 		const hasPricing = [model.cost, ...(model.cost.tiers ?? [])].some(
 			({ input: inputCost, output: outputCost, cacheRead, cacheWrite }) =>
 				inputCost > 0 || outputCost > 0 || cacheRead > 0 || cacheWrite > 0,
 		);
-		result = {
-			output,
+		runDiagnostics = {
 			events: toTranscriptEvents(session.messages),
 			metadata: { systemPromptSha256: createHash("sha256").update(systemPrompt).digest("hex") },
 			usage: {
@@ -394,6 +405,10 @@ async function runPiCodingAgent<TOutput extends JsonValue>(
 				},
 			},
 		};
+		verifySystemPrompt(session.messages, options);
+		const output =
+			"output" in options ? await options.output({ response, session, systemPrompt, agentDir }) : response;
+		result = { output, ...runDiagnostics };
 	} catch (error) {
 		runError = error;
 	} finally {
@@ -425,12 +440,25 @@ async function runPiCodingAgent<TOutput extends JsonValue>(
 		}
 	}
 
-	if (runError !== undefined) {
-		if (cleanupErrors.length === 0) throw runError;
-		throw new AggregateError([runError, ...cleanupErrors], "Agent run failed and cleanup also failed.");
+	let failure = runError;
+	if (runError !== undefined && cleanupErrors.length > 0) {
+		failure = new AggregateError([runError, ...cleanupErrors], "Agent run failed and cleanup also failed.");
+	} else if (cleanupErrors.length === 1) {
+		failure = cleanupErrors[0];
+	} else if (cleanupErrors.length > 1) {
+		failure = new AggregateError(cleanupErrors, "Agent cleanup failed.");
+	}
+	if (failure !== undefined) {
+		if (runDiagnostics) {
+			const partialRun = normalizeHarnessRun(input, {
+				...runDiagnostics,
+				errors: [failure],
+				timings: { totalMs: performance.now() - startedAt },
+			});
+			throw attachHarnessRunToError(failure, partialRun);
+		}
+		throw failure;
 	}
-	if (cleanupErrors.length === 1) throw cleanupErrors[0];
-	if (cleanupErrors.length > 1) throw new AggregateError(cleanupErrors, "Agent cleanup failed.");
 	if (!result) throw new Error("Pi eval completed without a result.");
 	return { ...result, timings: { totalMs: performance.now() - startedAt } };
 }

packages/evals/src/report.ts

@@ -150,7 +150,7 @@ export async function readTaskObservation(
 	const assertion = assertions[0];
 	if (assertion.fullName !== reportedFullName) return { ...identity, outcome: "errored" };
 	const statusOutcome = classifyCaseStatus(assertion.status);
-	if (statusOutcome) return { ...identity, outcome: statusOutcome };
+	if (statusOutcome === "skipped" || statusOutcome === "pending") return { ...identity, outcome: statusOutcome };
 	if (workspace.cases.length !== 1) return { ...identity, outcome: "errored" };
 	const caseResult = workspace.cases[0];
 	if (caseResult.fullName !== reportedFullName) return { ...identity, outcome: "errored" };
@@ -175,7 +175,9 @@ export async function readTaskObservation(
 	} catch {
 		return { ...identity, outcome: "errored" };
 	}
-	if (run.errors.length > 0) return { ...identity, ...metrics, outcome: "errored" };
+	if (statusOutcome === "errored" || run.errors.length > 0) {
+		return { ...identity, ...metrics, outcome: "errored" };
+	}
 	let score: number | undefined;
 	try {
 		score = validateScore(caseResult.eval?.avgScore);

packages/evals/test/harness.test.ts

@@ -1,4 +1,5 @@
 import { homedir } from "node:os";
+import type { Message } from "@earendil-works/pi-ai";
 import { getDocsPath, getExamplesPath, getReadmePath } from "@earendil-works/pi-coding-agent";
 import { describe, expect, it, vi } from "vitest";
 import { buildSystemPrompt } from "../../coding-agent/src/core/system-prompt.ts";
@@ -9,6 +10,7 @@ import {
 	excludePiDocumentation,
 	resolveDocumentationVariant,
 	resolveModelSelection,
+	verifySystemPrompt,
 } from "../src/harness.ts";
 
 describe("resolveModelSelection", () => {
@@ -89,6 +91,20 @@ describe("documentation variant", () => {
 		expect(stripped).not.toContain(getExamplesPath());
 	});
 
+	it("verifies the replayed prompt that was sent before a later reload", () => {
+		const prompt = buildSystemPrompt({
+			cwd: "/workspace",
+			selectedTools: [...DOCUMENTATION_EVAL_TOOLS],
+		});
+		const stripped = excludePiDocumentation(prompt);
+		const messages: Message[] = [
+			{ role: "system", content: stripped, replace: true, timestamp: 0 },
+			{ role: "user", content: [{ type: "text", text: "Configure Pi" }], timestamp: 1 },
+		];
+
+		expect(verifySystemPrompt(messages, { name: "without_docs", expectedPiDocumentation: false })).toBe(stripped);
+	});
+
 	it("fails closed when prompt markers are missing", () => {
 		expect(() => excludePiDocumentation("Instructions")).toThrow("no Pi documentation section");
 		expect(() => excludePiDocumentation("\n<docs>\nPi documentation\n</docs>")).toThrow(

packages/evals/test/report.test.ts

@@ -123,6 +123,33 @@ describe("readTaskObservation", () => {
 		expect(observation).toMatchObject({ outcome: "pending" });
 	});
 
+	it("preserves metrics from a failed eval with a partial harness run", async () => {
+		const { directory, observation } = await readObservation({
+			status: "failed",
+			meta: scoredMeta({ errors: [{ message: "Prompt verification failed" }] }),
+		});
+		expect(observation).toEqual({
+			evalSet: task.evalSet,
+			caseId: task.caseId,
+			variant: task.variant,
+			model: task.model,
+			runNumber: task.runNumber,
+			outcome: "errored",
+			inputTokens: 10,
+			outputTokens: 5,
+			cacheReadTokens: 2,
+			cacheWriteTokens: 3,
+			totalTokens: 15,
+			toolCalls: 1,
+			totalMs: 1234,
+			estimatedCostUsd: 0.01,
+		});
+		const hashes = await readdir(join(directory, task.variant, "sessions"));
+		await expect(
+			readFile(join(directory, task.variant, "sessions", hashes[0], "session.jsonl"), "utf8"),
+		).resolves.toBe(SESSION);
+	});
+
 	it("records an errored outcome when a passed eval has no harness run", async () => {
 		const { observation } = await readObservation({ status: "passed" });
 		expect(observation).toMatchObject({ outcome: "errored" });