fix(coding-agent): record eval artifacts with explicit tasks

packages/evals/README.md

@@ -43,12 +43,14 @@ to each `describeEval(...)` suite:
 import { expect } from "vitest";
 import { describeEval } from "vitest-evals";
 import { createPiCodingAgentHarness } from "./pi-harness.ts";
+import { recordEvalSessionArtifact } from "./vitest-evals/artifacts.ts";
 
 const harness = createPiCodingAgentHarness({ noTools: "all" });
 
 describeEval("Pi smoke", { harness }, (it) => {
-	it("answers a factual question", async ({ run }) => {
+	it("answers a factual question", async ({ run, task }) => {
 		const result = await run("What is the capital of France? Reply with only the city name.");
+		await recordEvalSessionArtifact(task, result);
 		expect(result.output).toBe("Paris");
 	});
 });
@@ -109,6 +111,7 @@ Harnesses may differ by prompt, tools, skills, model, or any other Pi configurat
 ```ts
 import { describe } from "vitest";
 import { createJudge, describeEval } from "vitest-evals";
+import { recordEvalSessionArtifact } from "./vitest-evals/artifacts.ts";
 import { evalHarnessTable } from "./vitest-evals/harness-table.ts";
 
 const TargetTaskJudge = createJudge<string, string>("TargetTaskJudge", ({ output }) => ({
@@ -126,8 +129,9 @@ const harnessTable = evalHarnessTable(
 
 describe.for(harnessTable)("$name repetition $repetition", ({ harness }) => {
 	describeEval("target skill effectiveness", { harness, judges: [TargetTaskJudge], judgeThreshold: null }, (it) => {
-		it("completes the target task", async ({ run }) => {
-			await run("Complete the target task.");
+		it("completes the target task", async ({ run, task }) => {
+			const result = await run("Complete the target task.");
+			await recordEvalSessionArtifact(task, result);
 		});
 	});
 });
@@ -137,6 +141,10 @@ Comparative suites should record correctness with deterministic or model-backed
 This keeps a low score as an observation instead of making the Vitest invocation fail. Use hard assertions only for
 suite invariants and infrastructure contracts. `expect.soft(...)` still fails the test and is not a scoring mechanism.
 
+Record session attachments with the explicit Vitest `task` after `run(...)` returns. The Pi harness snapshots the native
+session before deleting its temporary workspace, while `recordEvalSessionArtifact(...)` registers that snapshot through
+Vitest's custom-artifact API.
+
 Harness names must be stable and unique within an eval set. The grouping key combines repetition and seed with a
 non-empty string `input.id` when available, otherwise with a SHA-256 hash of strict canonical JSON input. Execution order
 is seeded and randomized, while harness declaration order determines comparison direction. For each matched input and

packages/evals/src/extensions.eval.ts

@@ -3,7 +3,7 @@ import { join } from "node:path";
 import { describe, expect } from "vitest";
 import { createJudge, describeEval } from "vitest-evals";
 import { createPiCodingAgentHarness, type PiCodingAgentInput } from "./pi-harness.ts";
-import { recordEvalExtensionSourceArtifact } from "./vitest-evals/artifacts.ts";
+import { recordEvalExtensionSourceArtifact, recordEvalSessionArtifact } from "./vitest-evals/artifacts.ts";
 import { evalHarnessTable } from "./vitest-evals/harness-table.ts";
 
 type ExtensionAuthoringOutput = {
@@ -125,6 +125,7 @@ describe.for(extensionHarnessTable)("$name", ({ harness }) => {
 							"Use the hello tool to greet Bob. Respond with exactly the tool's greeting and nothing else.",
 					},
 				]);
+				await recordEvalSessionArtifact(task, result);
 				if (result.output.extensionSource !== null) {
 					const runId = result.artifacts?.runId;
 					if (typeof runId !== "string") throw new Error("Pi eval run did not record a run ID.");

packages/evals/src/pi-harness.ts

@@ -1,6 +1,6 @@
 import assert from "node:assert";
 import { existsSync } from "node:fs";
-import { mkdir, mkdtemp, rm } from "node:fs/promises";
+import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { performance } from "node:perf_hooks";
@@ -24,7 +24,7 @@ import {
 	type TranscriptEvent,
 	toJsonValue,
 } from "vitest-evals/harness";
-import { getCurrentEvalTest, recordEvalFileArtifact } from "./vitest-evals/artifacts.ts";
+import { PI_SESSION_JSONL_RUN_ARTIFACT } from "./vitest-evals/artifact-names.ts";
 
 export type PiCodingAgentInput = string | Array<{ type: "prompt"; content: string } | { type: "reload" }>;
 
@@ -114,8 +114,6 @@ async function runPiCodingAgent<TOutput extends JsonValue>(
 	options: PiCodingAgentHarnessOptions | PiCodingAgentHarnessWithOutput<TOutput>,
 ): Promise<SimpleHarnessResult<string | TOutput>> {
 	const startedAt = performance.now();
-	const artifactDirectory = process.env.PI_EVAL_ARTIFACT_DIR?.trim();
-	const evalTest = artifactDirectory ? getCurrentEvalTest() : undefined;
 	signal?.throwIfAborted();
 	const selection = resolveModelSelection(options.model);
 	const modelRuntime = await ModelRuntime.create();
@@ -215,14 +213,11 @@ async function runPiCodingAgent<TOutput extends JsonValue>(
 	}
 
 	const cleanupErrors: unknown[] = [];
-	if (sessionManager && artifactDirectory && evalTest) {
+	if (sessionManager) {
 		try {
 			const sessionPath = sessionManager.getSessionFile();
 			if (sessionPath && existsSync(sessionPath)) {
-				await recordEvalFileArtifact(evalTest, sessionManager.getSessionId(), {
-					name: "session",
-					path: sessionPath,
-				});
+				setArtifact(PI_SESSION_JSONL_RUN_ARTIFACT, await readFile(sessionPath, "utf8"));
 			}
 		} catch (error) {
 			cleanupErrors.push(error);

packages/evals/src/smoke.eval.ts

@@ -1,12 +1,14 @@
 import { expect } from "vitest";
 import { describeEval } from "vitest-evals";
 import { createPiCodingAgentHarness } from "./pi-harness.ts";
+import { recordEvalSessionArtifact } from "./vitest-evals/artifacts.ts";
 
 const piCodingAgentHarness = createPiCodingAgentHarness({ noTools: "all" });
 
 describeEval("Pi Coding Agent smoke", { harness: piCodingAgentHarness }, (it) => {
-	it("runs a basic prompt end to end", async ({ run }) => {
+	it("runs a basic prompt end to end", async ({ run, task }) => {
 		const result = await run("What's the capital of France? Respond with only the city name.");
+		await recordEvalSessionArtifact(task, result);
 
 		expect(result.output.trim()).toBe("Paris");
 		expect(result.errors).toEqual([]);

packages/evals/src/vitest-evals/artifact-names.ts

@@ -0,0 +1 @@
+export const PI_SESSION_JSONL_RUN_ARTIFACT = "piSessionJsonl";

packages/evals/src/vitest-evals/artifacts.ts

@@ -1,64 +1,66 @@
 import { createHash } from "node:crypto";
 import { mkdir, writeFile } from "node:fs/promises";
-import { basename, isAbsolute, join, relative } from "node:path";
+import { basename, join, relative } from "node:path";
 import {
 	type RunnerTestCase,
 	recordArtifact,
 	type TestArtifact,
 	type TestArtifactBase,
 	type TestAttachment,
-	TestRunner,
 } from "vitest";
+import type { HarnessRun } from "vitest-evals/harness";
+import { PI_SESSION_JSONL_RUN_ARTIFACT } from "./artifact-names.ts";
 
-const evalFileArtifactKey = Symbol("pi-evals-file-artifact");
+const evalSessionArtifactKey = Symbol("pi-evals-session-artifact");
 const evalExtensionSourceArtifactKey = Symbol("pi-evals-extension-source-artifact");
 
-export interface EvalFileAttachment extends TestAttachment {
+interface EvalTextAttachment extends TestAttachment {
 	name: string;
-	path: string;
-}
-
-interface EvalExtensionSourceAttachment extends TestAttachment {
-	name: string;
-	contentType: "text/typescript";
+	contentType: string;
 	body: string;
 	bodyEncoding: "utf-8";
 }
 
-interface EvalFileArtifact extends TestArtifactBase {
-	type: "@earendil-works/pi-evals:file";
+interface EvalSessionArtifact extends TestArtifactBase {
+	type: "@earendil-works/pi-evals:session";
 	runId: string;
-	attachments: EvalFileAttachment[];
+	attachments: [EvalTextAttachment] | [];
 }
 
 interface EvalExtensionSourceArtifact extends TestArtifactBase {
 	type: "@earendil-works/pi-evals:extension-source";
 	runId: string;
-	attachments: [EvalExtensionSourceAttachment] | [];
+	attachments: [EvalTextAttachment] | [];
 }
 
 declare module "vitest" {
 	interface TestArtifactRegistry {
-		[evalFileArtifactKey]: EvalFileArtifact;
+		[evalSessionArtifactKey]: EvalSessionArtifact;
 		[evalExtensionSourceArtifactKey]: EvalExtensionSourceArtifact;
 	}
 }
 
-export function getCurrentEvalTest(): NonNullable<ReturnType<typeof TestRunner.getCurrentTest>> {
-	const test = TestRunner.getCurrentTest();
-	if (!test) throw new Error("Cannot record an eval artifact outside a running Vitest test.");
-	return test;
-}
-
-export async function recordEvalFileArtifact(
-	test: NonNullable<ReturnType<typeof TestRunner.getCurrentTest>>,
-	runId: string,
-	attachment: EvalFileAttachment,
+export async function recordEvalSessionArtifact(
+	task: Readonly<RunnerTestCase>,
+	run: Pick<HarnessRun, "artifacts">,
 ): Promise<void> {
-	await recordArtifact(test, {
-		type: "@earendil-works/pi-evals:file",
+	const runId = run.artifacts?.runId;
+	const session = run.artifacts?.[PI_SESSION_JSONL_RUN_ARTIFACT];
+	if (session === undefined) return;
+	if (typeof runId !== "string" || typeof session !== "string") {
+		throw new TypeError("Pi eval session artifact metadata is invalid.");
+	}
+	await recordArtifact(task, {
+		type: "@earendil-works/pi-evals:session",
 		runId,
-		attachments: [attachment],
+		attachments: [
+			{
+				name: "session.jsonl",
+				contentType: "application/x-ndjson",
+				body: session,
+				bodyEncoding: "utf-8",
+			},
+		],
 	});
 }
 
@@ -88,23 +90,22 @@ export async function persistEvalArtifactReferences(
 ): Promise<Array<{ name: string; path: string }>> {
 	const references: Array<{ name: string; path: string }> = [];
 	for (const artifact of artifacts) {
-		if (artifact.type === "@earendil-works/pi-evals:file" && artifact.runId === runId) {
-			for (const attachment of artifact.attachments) {
-				references.push({
-					name: attachment.name,
-					path: isAbsolute(attachment.path) ? relative(artifactDirectory, attachment.path) : attachment.path,
-				});
-			}
-		} else if (artifact.type === "@earendil-works/pi-evals:extension-source" && artifact.runId === runId) {
-			for (const attachment of artifact.attachments) {
-				const name = basename(attachment.name);
-				if (name !== attachment.name) throw new TypeError(`Invalid eval artifact name: ${attachment.name}`);
-				const directory = join(artifactDirectory, "sources", createHash("sha256").update(runId).digest("hex"));
-				await mkdir(directory, { recursive: true, mode: 0o700 });
-				const path = join(directory, name);
-				await writeFile(path, attachment.body, { encoding: "utf8", mode: 0o600 });
-				references.push({ name, path: relative(artifactDirectory, path) });
-			}
+		if (
+			(artifact.type !== "@earendil-works/pi-evals:session" &&
+				artifact.type !== "@earendil-works/pi-evals:extension-source") ||
+			artifact.runId !== runId
+		) {
+			continue;
+		}
+		const category = artifact.type === "@earendil-works/pi-evals:session" ? "sessions" : "sources";
+		for (const attachment of artifact.attachments) {
+			const name = basename(attachment.name);
+			if (name !== attachment.name) throw new TypeError(`Invalid eval artifact name: ${attachment.name}`);
+			const directory = join(artifactDirectory, category, createHash("sha256").update(runId).digest("hex"));
+			await mkdir(directory, { recursive: true, mode: 0o700 });
+			const path = join(directory, name);
+			await writeFile(path, attachment.body, { encoding: "utf8", mode: 0o600 });
+			references.push({ name, path: relative(artifactDirectory, path) });
 		}
 	}
 	return references;

packages/evals/src/vitest-evals/reporter.ts

@@ -3,6 +3,7 @@ import { appendFile, mkdir } from "node:fs/promises";
 import { join } from "node:path";
 import type { Reporter, SerializedError, TestCase, TestModule, TestRunEndReason, Vitest } from "vitest/node";
 import { isHarnessRun } from "vitest-evals/harness";
+import { PI_SESSION_JSONL_RUN_ARTIFACT } from "./artifact-names.ts";
 import { persistEvalArtifactReferences } from "./artifacts.ts";
 import { EVAL_HARNESS_ITERATION_ARTIFACT, parseEvalHarnessIterationArtifact } from "./harness-table.ts";
 import { formatHarnessComparisonReport, type HarnessObservation, summarizeHarnessComparisons } from "./summary.ts";
@@ -25,7 +26,11 @@ async function appendHarnessRunReport(test: TestCase): Promise<void> {
 	const run = harness.run;
 	const artifactRunId = run.artifacts?.runId;
 	const runId = typeof artifactRunId === "string" ? artifactRunId : randomUUID();
-	const metadata = Object.fromEntries(Object.entries(run.artifacts ?? {}).filter(([name]) => name !== "runId"));
+	const metadata = Object.fromEntries(
+		Object.entries(run.artifacts ?? {}).filter(
+			([name]) => name !== "runId" && name !== PI_SESSION_JSONL_RUN_ARTIFACT,
+		),
+	);
 	const record = {
 		schemaVersion: 1,
 		runId,

packages/evals/test/vitest-evals/artifacts.test.ts

@@ -1,53 +1,48 @@
-import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { mkdtemp, readFile, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { expect, it } from "vitest";
 import {
-	getCurrentEvalTest,
 	persistEvalArtifactReferences,
 	recordEvalExtensionSourceArtifact,
-	recordEvalFileArtifact,
+	recordEvalSessionArtifact,
 } from "../../src/vitest-evals/artifacts.ts";
 
-it("records file and inline Vitest attachments", async ({ task }) => {
-	const root = await mkdtemp(join(tmpdir(), "pi-eval-artifact-test-"));
-	try {
-		const runId = "run-1";
-		const sessionPath = join(root, "session.jsonl");
-		await writeFile(sessionPath, '{"type":"session"}\n', "utf8");
+it("records session and extension source artifacts against the explicit test task", async ({ task }) => {
+	const runId = "run-1";
+	await recordEvalSessionArtifact(task, {
+		artifacts: { runId, piSessionJsonl: '{"type":"session"}\n' },
+	});
+	await recordEvalExtensionSourceArtifact(task, runId, "export default function () {}\n");
 
-		const test = getCurrentEvalTest();
-		await recordEvalFileArtifact(test, runId, { name: "session", path: sessionPath });
-		await recordEvalExtensionSourceArtifact(task, runId, "export default function () {}\n");
-		expect(test.artifacts).toContainEqual(
-			expect.objectContaining({
-				type: "@earendil-works/pi-evals:file",
-				runId,
-				attachments: [
-					expect.objectContaining({
-						name: "session",
-						path: expect.any(String),
-					}),
-				],
-			}),
-		);
-		expect(test.artifacts).toContainEqual(
-			expect.objectContaining({
-				type: "@earendil-works/pi-evals:extension-source",
-				runId,
-				attachments: [
-					expect.objectContaining({
-						name: "hello.ts",
-						body: "export default function () {}\n",
-						bodyEncoding: "utf-8",
-						contentType: "text/typescript",
-					}),
-				],
-			}),
-		);
-	} finally {
-		await rm(root, { recursive: true, force: true });
-	}
+	expect(task.artifacts).toContainEqual(
+		expect.objectContaining({
+			type: "@earendil-works/pi-evals:session",
+			runId,
+			attachments: [
+				expect.objectContaining({
+					name: "session.jsonl",
+					body: '{"type":"session"}\n',
+					bodyEncoding: "utf-8",
+					contentType: "application/x-ndjson",
+				}),
+			],
+		}),
+	);
+	expect(task.artifacts).toContainEqual(
+		expect.objectContaining({
+			type: "@earendil-works/pi-evals:extension-source",
+			runId,
+			attachments: [
+				expect.objectContaining({
+					name: "hello.ts",
+					body: "export default function () {}\n",
+					bodyEncoding: "utf-8",
+					contentType: "text/typescript",
+				}),
+			],
+		}),
+	);
 });
 
 it("persists and selects attachments belonging to the reported run", async () => {
@@ -56,14 +51,21 @@ it("persists and selects attachments belonging to the reported run", async () =>
 		const references = await persistEvalArtifactReferences(
 			[
 				{
-					type: "@earendil-works/pi-evals:file",
+					type: "@earendil-works/pi-evals:session",
 					runId: "run-1",
-					attachments: [{ name: "session", path: join(root, "sessions", "run-1.jsonl") }],
+					attachments: [
+						{
+							name: "session.jsonl",
+							body: '{"type":"session"}\n',
+							bodyEncoding: "utf-8",
+							contentType: "application/x-ndjson",
+						},
+					],
 				},
 				{
-					type: "@earendil-works/pi-evals:file",
+					type: "@earendil-works/pi-evals:session",
 					runId: "run-2",
-					attachments: [{ name: "session", path: "/eval/sessions/run-2.jsonl" }],
+					attachments: [],
 				},
 				{
 					type: "@earendil-works/pi-evals:extension-source",
@@ -83,12 +85,13 @@ it("persists and selects attachments belonging to the reported run", async () =>
 			root,
 		);
 		expect(references).toEqual([
-			{ name: "session", path: "sessions/run-1.jsonl" },
+			{ name: "session.jsonl", path: expect.stringMatching(/^sessions\/[a-f0-9]{64}\/session\.jsonl$/) },
 			{ name: "hello.ts", path: expect.stringMatching(/^sources\/[a-f0-9]{64}\/hello\.ts$/) },
 		]);
-		const source = references.find(({ name }) => name === "hello.ts");
-		if (!source) throw new TypeError("Expected a persisted extension source reference.");
-		expect(await readFile(join(root, source.path), "utf8")).toBe("export default function () {}\n");
+		for (const { name, path } of references) {
+			const expected = name === "session.jsonl" ? '{"type":"session"}\n' : "export default function () {}\n";
+			expect(await readFile(join(root, path), "utf8")).toBe(expected);
+		}
 	} finally {
 		await rm(root, { recursive: true, force: true });
 	}

packages/evals/vitest.config.ts

@@ -1,4 +1,3 @@
-import { join } from "node:path";
 import { defineConfig, mergeConfig } from "vitest/config";
 import baseConfig, { workspaceSourcePaths } from "../../vitest.base.ts";
 
@@ -10,7 +9,6 @@ export default mergeConfig(
 			fileParallelism: false,
 			include: ["src/**/*.eval.ts"],
 			testTimeout: 120000,
-			attachmentsDir: join(process.env.PI_EVAL_ARTIFACT_DIR ?? ".eval", "sessions"),
 			hookTimeout: 30000,
 			reporters: ["vitest-evals/reporter", "./src/vitest-evals/reporter.ts"],
 		},

packages/evals/vitest.test.config.ts

@@ -6,7 +6,6 @@ export default mergeConfig(
 	defineConfig({
 		test: {
 			include: ["test/**/*.test.ts"],
-			attachmentsDir: ".eval/test-attachments",
 		},
 		resolve: {
 			alias: [{ find: /^@earendil-works\/pi-coding-agent$/, replacement: workspaceSourcePaths.codingAgentIndex }],