diff --git a/packages/evals/src/harness.ts b/packages/evals/src/harness.ts index 77ff8055f..f06f10aac 100644 --- a/packages/evals/src/harness.ts +++ b/packages/evals/src/harness.ts @@ -6,6 +6,7 @@ import { tmpdir } from "node:os"; import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { performance } from "node:perf_hooks"; import { contentText, InMemoryCredentialStore } from "@earendil-works/pi-ai"; +import { getCurrentSystemPrompt } from "@earendil-works/pi-ai/utils/transcript"; import { type AgentSession, type CreateAgentSessionOptions, @@ -18,18 +19,27 @@ import { SessionManager, } from "@earendil-works/pi-coding-agent"; import { + attachHarnessRunToError, createHarness, type Harness, type HarnessContext, type JsonValue, + normalizeHarnessRun, normalizeRecord, type SimpleHarnessResult, type TranscriptEvent, toJsonValue, + type UsageSummary, } from "vitest-evals/harness"; import type { DocumentationVariant } from "./plan.ts"; import { PI_SESSION_SNAPSHOT_ARTIFACT } from "./report.ts"; +type PiRunDiagnostics = { + events: TranscriptEvent[]; + metadata: Record; + usage: UsageSummary; +}; + export type PiCodingAgentInput = string | Array<{ type: "prompt"; content: string } | { type: "reload" }>; export type PiCodingAgentModelSelection = { @@ -244,8 +254,12 @@ async function promptAgent(session: AgentSession, input: string, signal: AbortSi return output ?? ""; } -function verifySystemPrompt(systemPrompt: string, options: PiCodingAgentHarnessOptions): void { - if (options.expectedPiDocumentation === undefined) return; +export function verifySystemPrompt( + messages: AgentSession["messages"], + options: Pick, +): string { + const systemPrompt = getCurrentSystemPrompt(messages); + if (options.expectedPiDocumentation === undefined) return systemPrompt; if (!systemPrompt.includes("\n\n")) { throw new Error(`Pi system prompt lost its rules in the ${options.name} eval variant.`); } @@ -253,6 +267,7 @@ function verifySystemPrompt(systemPrompt: string, options: PiCodingAgentHarnessO if (hasDocumentation !== options.expectedPiDocumentation) { throw new Error(`Pi system prompt does not match the ${options.name} eval variant.`); } + return systemPrompt; } async function runPiCodingAgent( @@ -285,6 +300,7 @@ async function runPiCodingAgent( let sessionManager: SessionManager | undefined; let session: AgentSession | undefined; let result: SimpleHarnessResult | undefined; + let runDiagnostics: PiRunDiagnostics | undefined; let runError: unknown; const cleanupErrors: unknown[] = []; let hiddenCredentialEnvironment: { name: string; value: string } | undefined; @@ -345,7 +361,6 @@ async function runPiCodingAgent( await enterToolSandbox(root, sandboxIdentity); let response: string | undefined; - let systemPrompt: string | undefined; const steps = typeof input === "string" ? [{ type: "prompt" as const, content: input }] : input; let abortPromise: Promise | undefined; const abort = () => { @@ -359,25 +374,21 @@ async function runPiCodingAgent( continue; } response = await promptAgent(session, step.content, signal); - systemPrompt = session.systemPrompt; } } finally { signal?.removeEventListener("abort", abort); if (abortPromise) await abortPromise; } - if (response === undefined || systemPrompt === undefined) { + if (response === undefined) { throw new Error("Pi eval input must include at least one prompt step."); } - verifySystemPrompt(systemPrompt, options); - const output = - "output" in options ? await options.output({ response, session, systemPrompt, agentDir }) : response; + const systemPrompt = getCurrentSystemPrompt(session.messages); const stats = session.getSessionStats(); const hasPricing = [model.cost, ...(model.cost.tiers ?? [])].some( ({ input: inputCost, output: outputCost, cacheRead, cacheWrite }) => inputCost > 0 || outputCost > 0 || cacheRead > 0 || cacheWrite > 0, ); - result = { - output, + runDiagnostics = { events: toTranscriptEvents(session.messages), metadata: { systemPromptSha256: createHash("sha256").update(systemPrompt).digest("hex") }, usage: { @@ -394,6 +405,10 @@ async function runPiCodingAgent( }, }, }; + verifySystemPrompt(session.messages, options); + const output = + "output" in options ? await options.output({ response, session, systemPrompt, agentDir }) : response; + result = { output, ...runDiagnostics }; } catch (error) { runError = error; } finally { @@ -425,12 +440,25 @@ async function runPiCodingAgent( } } - if (runError !== undefined) { - if (cleanupErrors.length === 0) throw runError; - throw new AggregateError([runError, ...cleanupErrors], "Agent run failed and cleanup also failed."); + let failure = runError; + if (runError !== undefined && cleanupErrors.length > 0) { + failure = new AggregateError([runError, ...cleanupErrors], "Agent run failed and cleanup also failed."); + } else if (cleanupErrors.length === 1) { + failure = cleanupErrors[0]; + } else if (cleanupErrors.length > 1) { + failure = new AggregateError(cleanupErrors, "Agent cleanup failed."); + } + if (failure !== undefined) { + if (runDiagnostics) { + const partialRun = normalizeHarnessRun(input, { + ...runDiagnostics, + errors: [failure], + timings: { totalMs: performance.now() - startedAt }, + }); + throw attachHarnessRunToError(failure, partialRun); + } + throw failure; } - if (cleanupErrors.length === 1) throw cleanupErrors[0]; - if (cleanupErrors.length > 1) throw new AggregateError(cleanupErrors, "Agent cleanup failed."); if (!result) throw new Error("Pi eval completed without a result."); return { ...result, timings: { totalMs: performance.now() - startedAt } }; } diff --git a/packages/evals/src/report.ts b/packages/evals/src/report.ts index 7c5471299..7c5e1d093 100644 --- a/packages/evals/src/report.ts +++ b/packages/evals/src/report.ts @@ -150,7 +150,7 @@ export async function readTaskObservation( const assertion = assertions[0]; if (assertion.fullName !== reportedFullName) return { ...identity, outcome: "errored" }; const statusOutcome = classifyCaseStatus(assertion.status); - if (statusOutcome) return { ...identity, outcome: statusOutcome }; + if (statusOutcome === "skipped" || statusOutcome === "pending") return { ...identity, outcome: statusOutcome }; if (workspace.cases.length !== 1) return { ...identity, outcome: "errored" }; const caseResult = workspace.cases[0]; if (caseResult.fullName !== reportedFullName) return { ...identity, outcome: "errored" }; @@ -175,7 +175,9 @@ export async function readTaskObservation( } catch { return { ...identity, outcome: "errored" }; } - if (run.errors.length > 0) return { ...identity, ...metrics, outcome: "errored" }; + if (statusOutcome === "errored" || run.errors.length > 0) { + return { ...identity, ...metrics, outcome: "errored" }; + } let score: number | undefined; try { score = validateScore(caseResult.eval?.avgScore); diff --git a/packages/evals/test/harness.test.ts b/packages/evals/test/harness.test.ts index b8effa224..bb6ab3033 100644 --- a/packages/evals/test/harness.test.ts +++ b/packages/evals/test/harness.test.ts @@ -1,4 +1,5 @@ import { homedir } from "node:os"; +import type { Message } from "@earendil-works/pi-ai"; import { getDocsPath, getExamplesPath, getReadmePath } from "@earendil-works/pi-coding-agent"; import { describe, expect, it, vi } from "vitest"; import { buildSystemPrompt } from "../../coding-agent/src/core/system-prompt.ts"; @@ -9,6 +10,7 @@ import { excludePiDocumentation, resolveDocumentationVariant, resolveModelSelection, + verifySystemPrompt, } from "../src/harness.ts"; describe("resolveModelSelection", () => { @@ -89,6 +91,20 @@ describe("documentation variant", () => { expect(stripped).not.toContain(getExamplesPath()); }); + it("verifies the replayed prompt that was sent before a later reload", () => { + const prompt = buildSystemPrompt({ + cwd: "/workspace", + selectedTools: [...DOCUMENTATION_EVAL_TOOLS], + }); + const stripped = excludePiDocumentation(prompt); + const messages: Message[] = [ + { role: "system", content: stripped, replace: true, timestamp: 0 }, + { role: "user", content: [{ type: "text", text: "Configure Pi" }], timestamp: 1 }, + ]; + + expect(verifySystemPrompt(messages, { name: "without_docs", expectedPiDocumentation: false })).toBe(stripped); + }); + it("fails closed when prompt markers are missing", () => { expect(() => excludePiDocumentation("Instructions")).toThrow("no Pi documentation section"); expect(() => excludePiDocumentation("\n\nPi documentation\n")).toThrow( diff --git a/packages/evals/test/report.test.ts b/packages/evals/test/report.test.ts index 69146a9fd..f872ea4f3 100644 --- a/packages/evals/test/report.test.ts +++ b/packages/evals/test/report.test.ts @@ -123,6 +123,33 @@ describe("readTaskObservation", () => { expect(observation).toMatchObject({ outcome: "pending" }); }); + it("preserves metrics from a failed eval with a partial harness run", async () => { + const { directory, observation } = await readObservation({ + status: "failed", + meta: scoredMeta({ errors: [{ message: "Prompt verification failed" }] }), + }); + expect(observation).toEqual({ + evalSet: task.evalSet, + caseId: task.caseId, + variant: task.variant, + model: task.model, + runNumber: task.runNumber, + outcome: "errored", + inputTokens: 10, + outputTokens: 5, + cacheReadTokens: 2, + cacheWriteTokens: 3, + totalTokens: 15, + toolCalls: 1, + totalMs: 1234, + estimatedCostUsd: 0.01, + }); + const hashes = await readdir(join(directory, task.variant, "sessions")); + await expect( + readFile(join(directory, task.variant, "sessions", hashes[0], "session.jsonl"), "utf8"), + ).resolves.toBe(SESSION); + }); + it("records an errored outcome when a passed eval has no harness run", async () => { const { observation } = await readObservation({ status: "passed" }); expect(observation).toMatchObject({ outcome: "errored" });