From 56e6a8a3fa6fe1c71f1106345f3f3dfa60f71d04 Mon Sep 17 00:00:00 2001 From: Christian Klotz Date: Thu, 30 Jul 2026 02:05:43 +0300 Subject: [PATCH] fix(coding-agent): record eval sessions after each test --- packages/evals/README.md | 15 +++++---------- packages/evals/src/extensions.eval.ts | 3 +-- packages/evals/src/smoke.eval.ts | 4 +--- packages/evals/src/vitest-evals/setup.ts | 8 ++++++++ packages/evals/vitest.config.ts | 1 + 5 files changed, 16 insertions(+), 15 deletions(-) create mode 100644 packages/evals/src/vitest-evals/setup.ts diff --git a/packages/evals/README.md b/packages/evals/README.md index dd8230a1f..d38978f6a 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -43,14 +43,12 @@ to each `describeEval(...)` suite: import { expect } from "vitest"; import { describeEval } from "vitest-evals"; import { createPiCodingAgentHarness } from "./pi-harness.ts"; -import { recordEvalSessionArtifact } from "./vitest-evals/artifacts.ts"; const harness = createPiCodingAgentHarness({ noTools: "all" }); describeEval("Pi smoke", { harness }, (it) => { - it("answers a factual question", async ({ run, task }) => { + it("answers a factual question", async ({ run }) => { const result = await run("What is the capital of France? Reply with only the city name."); - await recordEvalSessionArtifact(task, result); expect(result.output).toBe("Paris"); }); }); @@ -111,7 +109,6 @@ Harnesses may differ by prompt, tools, skills, model, or any other Pi configurat ```ts import { describe } from "vitest"; import { createJudge, describeEval } from "vitest-evals"; -import { recordEvalSessionArtifact } from "./vitest-evals/artifacts.ts"; import { evalHarnessTable } from "./vitest-evals/harness-table.ts"; const TargetTaskJudge = createJudge("TargetTaskJudge", ({ output }) => ({ @@ -129,9 +126,8 @@ const harnessTable = evalHarnessTable( describe.for(harnessTable)("$name repetition $repetition", ({ harness }) => { describeEval("target skill effectiveness", { harness, judges: [TargetTaskJudge], judgeThreshold: null }, (it) => { - it("completes the target task", async ({ run, task }) => { - const result = await run("Complete the target task."); - await recordEvalSessionArtifact(task, result); + it("completes the target task", async ({ run }) => { + await run("Complete the target task."); }); }); }); @@ -141,9 +137,8 @@ Comparative suites should record correctness with deterministic or model-backed This keeps a low score as an observation instead of making the Vitest invocation fail. Use hard assertions only for suite invariants and infrastructure contracts. `expect.soft(...)` still fails the test and is not a scoring mechanism. -Record session attachments with the explicit Vitest `task` after `run(...)` returns. The Pi harness snapshots the native -session before deleting its temporary workspace, while `recordEvalSessionArtifact(...)` registers that snapshot through -Vitest's custom-artifact API. +The Pi harness snapshots native session JSONL before deleting its temporary workspace. An eval-only `afterEach` hook +registers that snapshot against the explicit Vitest test task before reporters run. Harness names must be stable and unique within an eval set. The grouping key combines repetition and seed with a non-empty string `input.id` when available, otherwise with a SHA-256 hash of strict canonical JSON input. Execution order diff --git a/packages/evals/src/extensions.eval.ts b/packages/evals/src/extensions.eval.ts index 8d30f977b..1e0a0caaf 100644 --- a/packages/evals/src/extensions.eval.ts +++ b/packages/evals/src/extensions.eval.ts @@ -3,7 +3,7 @@ import { join } from "node:path"; import { describe, expect } from "vitest"; import { createJudge, describeEval } from "vitest-evals"; import { createPiCodingAgentHarness, type PiCodingAgentInput } from "./pi-harness.ts"; -import { recordEvalSessionArtifact, recordEvalSourceArtifact } from "./vitest-evals/artifacts.ts"; +import { recordEvalSourceArtifact } from "./vitest-evals/artifacts.ts"; import { evalHarnessTable } from "./vitest-evals/harness-table.ts"; type ExtensionAuthoringOutput = { @@ -125,7 +125,6 @@ describe.for(extensionHarnessTable)("$name", ({ harness }) => { "Use the hello tool to greet Bob. Respond with exactly the tool's greeting and nothing else.", }, ]); - await recordEvalSessionArtifact(task, result); if (result.output.extensionSource !== null) { const runId = result.artifacts?.runId; if (typeof runId !== "string") throw new Error("Pi eval run did not record a run ID."); diff --git a/packages/evals/src/smoke.eval.ts b/packages/evals/src/smoke.eval.ts index b9367b6dc..3dd8920a9 100644 --- a/packages/evals/src/smoke.eval.ts +++ b/packages/evals/src/smoke.eval.ts @@ -1,14 +1,12 @@ import { expect } from "vitest"; import { describeEval } from "vitest-evals"; import { createPiCodingAgentHarness } from "./pi-harness.ts"; -import { recordEvalSessionArtifact } from "./vitest-evals/artifacts.ts"; const piCodingAgentHarness = createPiCodingAgentHarness({ noTools: "all" }); describeEval("Pi Coding Agent smoke", { harness: piCodingAgentHarness }, (it) => { - it("runs a basic prompt end to end", async ({ run, task }) => { + it("runs a basic prompt end to end", async ({ run }) => { const result = await run("What's the capital of France? Respond with only the city name."); - await recordEvalSessionArtifact(task, result); expect(result.output.trim()).toBe("Paris"); expect(result.errors).toEqual([]); diff --git a/packages/evals/src/vitest-evals/setup.ts b/packages/evals/src/vitest-evals/setup.ts new file mode 100644 index 000000000..a711fea9b --- /dev/null +++ b/packages/evals/src/vitest-evals/setup.ts @@ -0,0 +1,8 @@ +import { afterEach } from "vitest"; +import type {} from "vitest-evals"; +import { recordEvalSessionArtifact } from "./artifacts.ts"; + +afterEach(async ({ task }) => { + const run = task.meta.harness?.run; + if (run) await recordEvalSessionArtifact(task, run); +}); diff --git a/packages/evals/vitest.config.ts b/packages/evals/vitest.config.ts index 7c4fa0d2f..67dad900b 100644 --- a/packages/evals/vitest.config.ts +++ b/packages/evals/vitest.config.ts @@ -10,6 +10,7 @@ export default mergeConfig( include: ["src/**/*.eval.ts"], testTimeout: 120000, hookTimeout: 30000, + setupFiles: ["./src/vitest-evals/setup.ts"], reporters: ["vitest-evals/reporter", "./src/vitest-evals/reporter.ts"], }, resolve: {