diff --git a/packages/evals/README.md b/packages/evals/README.md index 9145a9e3a..dd8230a1f 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -43,12 +43,14 @@ to each `describeEval(...)` suite: import { expect } from "vitest"; import { describeEval } from "vitest-evals"; import { createPiCodingAgentHarness } from "./pi-harness.ts"; +import { recordEvalSessionArtifact } from "./vitest-evals/artifacts.ts"; const harness = createPiCodingAgentHarness({ noTools: "all" }); describeEval("Pi smoke", { harness }, (it) => { - it("answers a factual question", async ({ run }) => { + it("answers a factual question", async ({ run, task }) => { const result = await run("What is the capital of France? Reply with only the city name."); + await recordEvalSessionArtifact(task, result); expect(result.output).toBe("Paris"); }); }); @@ -109,6 +111,7 @@ Harnesses may differ by prompt, tools, skills, model, or any other Pi configurat ```ts import { describe } from "vitest"; import { createJudge, describeEval } from "vitest-evals"; +import { recordEvalSessionArtifact } from "./vitest-evals/artifacts.ts"; import { evalHarnessTable } from "./vitest-evals/harness-table.ts"; const TargetTaskJudge = createJudge("TargetTaskJudge", ({ output }) => ({ @@ -126,8 +129,9 @@ const harnessTable = evalHarnessTable( describe.for(harnessTable)("$name repetition $repetition", ({ harness }) => { describeEval("target skill effectiveness", { harness, judges: [TargetTaskJudge], judgeThreshold: null }, (it) => { - it("completes the target task", async ({ run }) => { - await run("Complete the target task."); + it("completes the target task", async ({ run, task }) => { + const result = await run("Complete the target task."); + await recordEvalSessionArtifact(task, result); }); }); }); @@ -137,6 +141,10 @@ Comparative suites should record correctness with deterministic or model-backed This keeps a low score as an observation instead of making the Vitest invocation fail. Use hard assertions only for suite invariants and infrastructure contracts. `expect.soft(...)` still fails the test and is not a scoring mechanism. +Record session attachments with the explicit Vitest `task` after `run(...)` returns. The Pi harness snapshots the native +session before deleting its temporary workspace, while `recordEvalSessionArtifact(...)` registers that snapshot through +Vitest's custom-artifact API. + Harness names must be stable and unique within an eval set. The grouping key combines repetition and seed with a non-empty string `input.id` when available, otherwise with a SHA-256 hash of strict canonical JSON input. Execution order is seeded and randomized, while harness declaration order determines comparison direction. For each matched input and diff --git a/packages/evals/src/extensions.eval.ts b/packages/evals/src/extensions.eval.ts index 87c2e3fc1..d3a913933 100644 --- a/packages/evals/src/extensions.eval.ts +++ b/packages/evals/src/extensions.eval.ts @@ -3,7 +3,7 @@ import { join } from "node:path"; import { describe, expect } from "vitest"; import { createJudge, describeEval } from "vitest-evals"; import { createPiCodingAgentHarness, type PiCodingAgentInput } from "./pi-harness.ts"; -import { recordEvalExtensionSourceArtifact } from "./vitest-evals/artifacts.ts"; +import { recordEvalExtensionSourceArtifact, recordEvalSessionArtifact } from "./vitest-evals/artifacts.ts"; import { evalHarnessTable } from "./vitest-evals/harness-table.ts"; type ExtensionAuthoringOutput = { @@ -125,6 +125,7 @@ describe.for(extensionHarnessTable)("$name", ({ harness }) => { "Use the hello tool to greet Bob. Respond with exactly the tool's greeting and nothing else.", }, ]); + await recordEvalSessionArtifact(task, result); if (result.output.extensionSource !== null) { const runId = result.artifacts?.runId; if (typeof runId !== "string") throw new Error("Pi eval run did not record a run ID."); diff --git a/packages/evals/src/pi-harness.ts b/packages/evals/src/pi-harness.ts index 61b4080a9..84177f67b 100644 --- a/packages/evals/src/pi-harness.ts +++ b/packages/evals/src/pi-harness.ts @@ -1,6 +1,6 @@ import assert from "node:assert"; import { existsSync } from "node:fs"; -import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { performance } from "node:perf_hooks"; @@ -24,7 +24,7 @@ import { type TranscriptEvent, toJsonValue, } from "vitest-evals/harness"; -import { getCurrentEvalTest, recordEvalFileArtifact } from "./vitest-evals/artifacts.ts"; +import { PI_SESSION_JSONL_RUN_ARTIFACT } from "./vitest-evals/artifact-names.ts"; export type PiCodingAgentInput = string | Array<{ type: "prompt"; content: string } | { type: "reload" }>; @@ -114,8 +114,6 @@ async function runPiCodingAgent( options: PiCodingAgentHarnessOptions | PiCodingAgentHarnessWithOutput, ): Promise> { const startedAt = performance.now(); - const artifactDirectory = process.env.PI_EVAL_ARTIFACT_DIR?.trim(); - const evalTest = artifactDirectory ? getCurrentEvalTest() : undefined; signal?.throwIfAborted(); const selection = resolveModelSelection(options.model); const modelRuntime = await ModelRuntime.create(); @@ -215,14 +213,11 @@ async function runPiCodingAgent( } const cleanupErrors: unknown[] = []; - if (sessionManager && artifactDirectory && evalTest) { + if (sessionManager) { try { const sessionPath = sessionManager.getSessionFile(); if (sessionPath && existsSync(sessionPath)) { - await recordEvalFileArtifact(evalTest, sessionManager.getSessionId(), { - name: "session", - path: sessionPath, - }); + setArtifact(PI_SESSION_JSONL_RUN_ARTIFACT, await readFile(sessionPath, "utf8")); } } catch (error) { cleanupErrors.push(error); diff --git a/packages/evals/src/smoke.eval.ts b/packages/evals/src/smoke.eval.ts index 3dd8920a9..b9367b6dc 100644 --- a/packages/evals/src/smoke.eval.ts +++ b/packages/evals/src/smoke.eval.ts @@ -1,12 +1,14 @@ import { expect } from "vitest"; import { describeEval } from "vitest-evals"; import { createPiCodingAgentHarness } from "./pi-harness.ts"; +import { recordEvalSessionArtifact } from "./vitest-evals/artifacts.ts"; const piCodingAgentHarness = createPiCodingAgentHarness({ noTools: "all" }); describeEval("Pi Coding Agent smoke", { harness: piCodingAgentHarness }, (it) => { - it("runs a basic prompt end to end", async ({ run }) => { + it("runs a basic prompt end to end", async ({ run, task }) => { const result = await run("What's the capital of France? Respond with only the city name."); + await recordEvalSessionArtifact(task, result); expect(result.output.trim()).toBe("Paris"); expect(result.errors).toEqual([]); diff --git a/packages/evals/src/vitest-evals/artifact-names.ts b/packages/evals/src/vitest-evals/artifact-names.ts new file mode 100644 index 000000000..aeed6541e --- /dev/null +++ b/packages/evals/src/vitest-evals/artifact-names.ts @@ -0,0 +1 @@ +export const PI_SESSION_JSONL_RUN_ARTIFACT = "piSessionJsonl"; diff --git a/packages/evals/src/vitest-evals/artifacts.ts b/packages/evals/src/vitest-evals/artifacts.ts index f8c2ae9f1..ea2585905 100644 --- a/packages/evals/src/vitest-evals/artifacts.ts +++ b/packages/evals/src/vitest-evals/artifacts.ts @@ -1,64 +1,66 @@ import { createHash } from "node:crypto"; import { mkdir, writeFile } from "node:fs/promises"; -import { basename, isAbsolute, join, relative } from "node:path"; +import { basename, join, relative } from "node:path"; import { type RunnerTestCase, recordArtifact, type TestArtifact, type TestArtifactBase, type TestAttachment, - TestRunner, } from "vitest"; +import type { HarnessRun } from "vitest-evals/harness"; +import { PI_SESSION_JSONL_RUN_ARTIFACT } from "./artifact-names.ts"; -const evalFileArtifactKey = Symbol("pi-evals-file-artifact"); +const evalSessionArtifactKey = Symbol("pi-evals-session-artifact"); const evalExtensionSourceArtifactKey = Symbol("pi-evals-extension-source-artifact"); -export interface EvalFileAttachment extends TestAttachment { +interface EvalTextAttachment extends TestAttachment { name: string; - path: string; -} - -interface EvalExtensionSourceAttachment extends TestAttachment { - name: string; - contentType: "text/typescript"; + contentType: string; body: string; bodyEncoding: "utf-8"; } -interface EvalFileArtifact extends TestArtifactBase { - type: "@earendil-works/pi-evals:file"; +interface EvalSessionArtifact extends TestArtifactBase { + type: "@earendil-works/pi-evals:session"; runId: string; - attachments: EvalFileAttachment[]; + attachments: [EvalTextAttachment] | []; } interface EvalExtensionSourceArtifact extends TestArtifactBase { type: "@earendil-works/pi-evals:extension-source"; runId: string; - attachments: [EvalExtensionSourceAttachment] | []; + attachments: [EvalTextAttachment] | []; } declare module "vitest" { interface TestArtifactRegistry { - [evalFileArtifactKey]: EvalFileArtifact; + [evalSessionArtifactKey]: EvalSessionArtifact; [evalExtensionSourceArtifactKey]: EvalExtensionSourceArtifact; } } -export function getCurrentEvalTest(): NonNullable> { - const test = TestRunner.getCurrentTest(); - if (!test) throw new Error("Cannot record an eval artifact outside a running Vitest test."); - return test; -} - -export async function recordEvalFileArtifact( - test: NonNullable>, - runId: string, - attachment: EvalFileAttachment, +export async function recordEvalSessionArtifact( + task: Readonly, + run: Pick, ): Promise { - await recordArtifact(test, { - type: "@earendil-works/pi-evals:file", + const runId = run.artifacts?.runId; + const session = run.artifacts?.[PI_SESSION_JSONL_RUN_ARTIFACT]; + if (session === undefined) return; + if (typeof runId !== "string" || typeof session !== "string") { + throw new TypeError("Pi eval session artifact metadata is invalid."); + } + await recordArtifact(task, { + type: "@earendil-works/pi-evals:session", runId, - attachments: [attachment], + attachments: [ + { + name: "session.jsonl", + contentType: "application/x-ndjson", + body: session, + bodyEncoding: "utf-8", + }, + ], }); } @@ -88,23 +90,22 @@ export async function persistEvalArtifactReferences( ): Promise> { const references: Array<{ name: string; path: string }> = []; for (const artifact of artifacts) { - if (artifact.type === "@earendil-works/pi-evals:file" && artifact.runId === runId) { - for (const attachment of artifact.attachments) { - references.push({ - name: attachment.name, - path: isAbsolute(attachment.path) ? relative(artifactDirectory, attachment.path) : attachment.path, - }); - } - } else if (artifact.type === "@earendil-works/pi-evals:extension-source" && artifact.runId === runId) { - for (const attachment of artifact.attachments) { - const name = basename(attachment.name); - if (name !== attachment.name) throw new TypeError(`Invalid eval artifact name: ${attachment.name}`); - const directory = join(artifactDirectory, "sources", createHash("sha256").update(runId).digest("hex")); - await mkdir(directory, { recursive: true, mode: 0o700 }); - const path = join(directory, name); - await writeFile(path, attachment.body, { encoding: "utf8", mode: 0o600 }); - references.push({ name, path: relative(artifactDirectory, path) }); - } + if ( + (artifact.type !== "@earendil-works/pi-evals:session" && + artifact.type !== "@earendil-works/pi-evals:extension-source") || + artifact.runId !== runId + ) { + continue; + } + const category = artifact.type === "@earendil-works/pi-evals:session" ? "sessions" : "sources"; + for (const attachment of artifact.attachments) { + const name = basename(attachment.name); + if (name !== attachment.name) throw new TypeError(`Invalid eval artifact name: ${attachment.name}`); + const directory = join(artifactDirectory, category, createHash("sha256").update(runId).digest("hex")); + await mkdir(directory, { recursive: true, mode: 0o700 }); + const path = join(directory, name); + await writeFile(path, attachment.body, { encoding: "utf8", mode: 0o600 }); + references.push({ name, path: relative(artifactDirectory, path) }); } } return references; diff --git a/packages/evals/src/vitest-evals/reporter.ts b/packages/evals/src/vitest-evals/reporter.ts index 6db498ae5..5b5058324 100644 --- a/packages/evals/src/vitest-evals/reporter.ts +++ b/packages/evals/src/vitest-evals/reporter.ts @@ -3,6 +3,7 @@ import { appendFile, mkdir } from "node:fs/promises"; import { join } from "node:path"; import type { Reporter, SerializedError, TestCase, TestModule, TestRunEndReason, Vitest } from "vitest/node"; import { isHarnessRun } from "vitest-evals/harness"; +import { PI_SESSION_JSONL_RUN_ARTIFACT } from "./artifact-names.ts"; import { persistEvalArtifactReferences } from "./artifacts.ts"; import { EVAL_HARNESS_ITERATION_ARTIFACT, parseEvalHarnessIterationArtifact } from "./harness-table.ts"; import { formatHarnessComparisonReport, type HarnessObservation, summarizeHarnessComparisons } from "./summary.ts"; @@ -25,7 +26,11 @@ async function appendHarnessRunReport(test: TestCase): Promise { const run = harness.run; const artifactRunId = run.artifacts?.runId; const runId = typeof artifactRunId === "string" ? artifactRunId : randomUUID(); - const metadata = Object.fromEntries(Object.entries(run.artifacts ?? {}).filter(([name]) => name !== "runId")); + const metadata = Object.fromEntries( + Object.entries(run.artifacts ?? {}).filter( + ([name]) => name !== "runId" && name !== PI_SESSION_JSONL_RUN_ARTIFACT, + ), + ); const record = { schemaVersion: 1, runId, diff --git a/packages/evals/test/vitest-evals/artifacts.test.ts b/packages/evals/test/vitest-evals/artifacts.test.ts index 3f3f5652d..9119eb844 100644 --- a/packages/evals/test/vitest-evals/artifacts.test.ts +++ b/packages/evals/test/vitest-evals/artifacts.test.ts @@ -1,53 +1,48 @@ -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { expect, it } from "vitest"; import { - getCurrentEvalTest, persistEvalArtifactReferences, recordEvalExtensionSourceArtifact, - recordEvalFileArtifact, + recordEvalSessionArtifact, } from "../../src/vitest-evals/artifacts.ts"; -it("records file and inline Vitest attachments", async ({ task }) => { - const root = await mkdtemp(join(tmpdir(), "pi-eval-artifact-test-")); - try { - const runId = "run-1"; - const sessionPath = join(root, "session.jsonl"); - await writeFile(sessionPath, '{"type":"session"}\n', "utf8"); +it("records session and extension source artifacts against the explicit test task", async ({ task }) => { + const runId = "run-1"; + await recordEvalSessionArtifact(task, { + artifacts: { runId, piSessionJsonl: '{"type":"session"}\n' }, + }); + await recordEvalExtensionSourceArtifact(task, runId, "export default function () {}\n"); - const test = getCurrentEvalTest(); - await recordEvalFileArtifact(test, runId, { name: "session", path: sessionPath }); - await recordEvalExtensionSourceArtifact(task, runId, "export default function () {}\n"); - expect(test.artifacts).toContainEqual( - expect.objectContaining({ - type: "@earendil-works/pi-evals:file", - runId, - attachments: [ - expect.objectContaining({ - name: "session", - path: expect.any(String), - }), - ], - }), - ); - expect(test.artifacts).toContainEqual( - expect.objectContaining({ - type: "@earendil-works/pi-evals:extension-source", - runId, - attachments: [ - expect.objectContaining({ - name: "hello.ts", - body: "export default function () {}\n", - bodyEncoding: "utf-8", - contentType: "text/typescript", - }), - ], - }), - ); - } finally { - await rm(root, { recursive: true, force: true }); - } + expect(task.artifacts).toContainEqual( + expect.objectContaining({ + type: "@earendil-works/pi-evals:session", + runId, + attachments: [ + expect.objectContaining({ + name: "session.jsonl", + body: '{"type":"session"}\n', + bodyEncoding: "utf-8", + contentType: "application/x-ndjson", + }), + ], + }), + ); + expect(task.artifacts).toContainEqual( + expect.objectContaining({ + type: "@earendil-works/pi-evals:extension-source", + runId, + attachments: [ + expect.objectContaining({ + name: "hello.ts", + body: "export default function () {}\n", + bodyEncoding: "utf-8", + contentType: "text/typescript", + }), + ], + }), + ); }); it("persists and selects attachments belonging to the reported run", async () => { @@ -56,14 +51,21 @@ it("persists and selects attachments belonging to the reported run", async () => const references = await persistEvalArtifactReferences( [ { - type: "@earendil-works/pi-evals:file", + type: "@earendil-works/pi-evals:session", runId: "run-1", - attachments: [{ name: "session", path: join(root, "sessions", "run-1.jsonl") }], + attachments: [ + { + name: "session.jsonl", + body: '{"type":"session"}\n', + bodyEncoding: "utf-8", + contentType: "application/x-ndjson", + }, + ], }, { - type: "@earendil-works/pi-evals:file", + type: "@earendil-works/pi-evals:session", runId: "run-2", - attachments: [{ name: "session", path: "/eval/sessions/run-2.jsonl" }], + attachments: [], }, { type: "@earendil-works/pi-evals:extension-source", @@ -83,12 +85,13 @@ it("persists and selects attachments belonging to the reported run", async () => root, ); expect(references).toEqual([ - { name: "session", path: "sessions/run-1.jsonl" }, + { name: "session.jsonl", path: expect.stringMatching(/^sessions\/[a-f0-9]{64}\/session\.jsonl$/) }, { name: "hello.ts", path: expect.stringMatching(/^sources\/[a-f0-9]{64}\/hello\.ts$/) }, ]); - const source = references.find(({ name }) => name === "hello.ts"); - if (!source) throw new TypeError("Expected a persisted extension source reference."); - expect(await readFile(join(root, source.path), "utf8")).toBe("export default function () {}\n"); + for (const { name, path } of references) { + const expected = name === "session.jsonl" ? '{"type":"session"}\n' : "export default function () {}\n"; + expect(await readFile(join(root, path), "utf8")).toBe(expected); + } } finally { await rm(root, { recursive: true, force: true }); } diff --git a/packages/evals/vitest.config.ts b/packages/evals/vitest.config.ts index dac0b81c2..7c4fa0d2f 100644 --- a/packages/evals/vitest.config.ts +++ b/packages/evals/vitest.config.ts @@ -1,4 +1,3 @@ -import { join } from "node:path"; import { defineConfig, mergeConfig } from "vitest/config"; import baseConfig, { workspaceSourcePaths } from "../../vitest.base.ts"; @@ -10,7 +9,6 @@ export default mergeConfig( fileParallelism: false, include: ["src/**/*.eval.ts"], testTimeout: 120000, - attachmentsDir: join(process.env.PI_EVAL_ARTIFACT_DIR ?? ".eval", "sessions"), hookTimeout: 30000, reporters: ["vitest-evals/reporter", "./src/vitest-evals/reporter.ts"], }, diff --git a/packages/evals/vitest.test.config.ts b/packages/evals/vitest.test.config.ts index eeec0c328..20aa79309 100644 --- a/packages/evals/vitest.test.config.ts +++ b/packages/evals/vitest.test.config.ts @@ -6,7 +6,6 @@ export default mergeConfig( defineConfig({ test: { include: ["test/**/*.test.ts"], - attachmentsDir: ".eval/test-attachments", }, resolve: { alias: [{ find: /^@earendil-works\/pi-coding-agent$/, replacement: workspaceSourcePaths.codingAgentIndex }],