feat(coding-agent): add prompt template documentation eval (#10261)

## Summary

- add live documentation comparisons for project-scoped and user-scoped
`/current-time` prompt templates
- require the generated command to expand to the exact requested prompt
text
- select the requested case from multi-case Vitest reports so skipped
siblings do not invalidate observations
- strengthen documentation-audit evidence and move the expensive audit
behind an explicit `eval:audit` command
- remove redundant eval oracles and test-only exports found during the
eval-suite audit

## Testing

- `npm run check`
- `./test.sh`
- `npm --prefix packages/evals test` (54 passed, also covered by
`./test.sh`)
- failed-status mutation: focused report test passed before the fixture
repair, failed after it, then passed with production restored
- prompt-template documentation eval, three repetitions per variant (6/6
pairs eligible)
- project scope: with docs 66.7%, without docs 100%; negative delta and
treatment flakiness
  - user scope: with docs 100%, without docs 100%; no lift
- model documentation eval (1/1 pair eligible; treatment 100%, control
0%)
- representative live documentation audit (`cli.md` passed)

packages/evals/README.md

@@ -7,7 +7,7 @@ Behavioral evals for Pi's coding agent, built with `vitest-evals`.
 Eval definitions are flat under `evals/`:
 
 - `*.docs.eval.ts` is a documentation-lift eval. `eval:docs` runs each case in isolated `without_docs` and `with_docs` containers and reports lift.
-- Other `*.eval.ts` files are host evals. `eval:host` runs them with Vitest on this machine. They are ordinary vitest-evals suites, not paired comparisons.
+- Other `*.eval.ts` files are host evals. `eval:host` runs the smoke suite; `eval:audit` runs the documentation audit. They are ordinary vitest-evals suites, not paired comparisons.
 
 Runner code lives in `src/`:
 
@@ -21,25 +21,25 @@ Eval suites and their fixtures live under `evals/`. Image build files live in `d
 
 ## Run evals
 
-Host evals (smoke, documentation audit) and documentation-lift evals need `PI_PROVIDER` and `PI_MODEL`.
+The host smoke eval, on-demand documentation audit, and documentation-lift evals need `PI_PROVIDER` and `PI_MODEL`.
 
 ```bash
 PI_PROVIDER=openai-codex PI_MODEL=gpt-5.6-sol npm run eval -w packages/evals
 ```
 
-That runs host evals, then the documentation comparison. Extra CLI flags after `--` go to `eval:docs` only.
+That runs the host smoke eval, then the documentation comparison. The documentation audit remains explicit because it scans every documentation page. Extra CLI flags after `--` go to `eval:docs` only.
 
-Host only:
+Host smoke only:
 
 ```bash
 PI_PROVIDER=openai-codex PI_MODEL=gpt-5.6-sol npm run eval:host -w packages/evals
 ```
 
-One host suite:
+Documentation audit:
 
 ```bash
 PI_PROVIDER=openai-codex PI_MODEL=gpt-5.6-sol \
-  npm run eval:host -w packages/evals -- evals/documentation-audit.eval.ts
+  npm run eval:audit -w packages/evals
 ```
 
 ## Run documentation comparisons

packages/evals/evals/configured-runtime.ts

@@ -1,4 +1,3 @@
-import { join } from "node:path";
 import { type Api, type Context, contentText, type Model, type ModelsSimpleStreamOptions } from "@earendil-works/pi-ai";
 import { ModelRuntime } from "@earendil-works/pi-coding-agent";
 
@@ -53,15 +52,6 @@ function modelFields(model: Model<Api>): ModelFields {
 	};
 }
 
-export function loadConfiguredModelRuntime(agentDir: string): Promise<ModelRuntime> {
-	return ModelRuntime.create({
-		modelsPath: join(agentDir, "models.json"),
-		authPath: join(agentDir, "auth.json"),
-		modelsStorePath: join(agentDir, "models-store.json"),
-		allowModelNetwork: false,
-	});
-}
-
 export async function inspectProvider(
 	runtime: ModelRuntime,
 	scenario: ProviderScenario,

packages/evals/evals/documentation-audit.eval.ts

@@ -1,5 +1,5 @@
 import { globSync } from "node:fs";
-import { resolve } from "node:path";
+import { isAbsolute, relative, resolve } from "node:path";
 import { Type } from "@earendil-works/pi-ai";
 import { defineTool } from "@earendil-works/pi-coding-agent";
 import { expect } from "vitest";
@@ -31,6 +31,7 @@ const submitAudit = defineTool({
 
 const repositoryRoot = resolve(import.meta.dirname, "../../..");
 const docsRoot = resolve(repositoryRoot, "packages/coding-agent/docs");
+const sourceRoot = resolve(repositoryRoot, "packages/coding-agent/src");
 const pages = globSync("**/*.md", { cwd: docsRoot })
 	.map((path) => ({ path: path.replaceAll("\\", "/") }))
 	.sort((left, right) => left.path.localeCompare(right.path));
@@ -56,7 +57,20 @@ For a mismatch, quote the claim, cite the implementation path and symbol, and st
 
 Call ${TOOL_NAME} exactly once as your final action. Do not return prose.`);
 
-		const auditCalls = toolCalls(result).filter((call) => call.name === TOOL_NAME);
+		const calls = toolCalls(result);
+		const readPaths = calls.flatMap((call) => {
+			const path = call.name === "read" && call.status === "ok" ? call.arguments?.path : undefined;
+			return typeof path === "string" ? [resolve(path)] : [];
+		});
+		expect(readPaths).toContain(documentationPath);
+		expect(
+			readPaths.some((readPath) => {
+				const sourceRelative = relative(sourceRoot, readPath);
+				return sourceRelative !== "" && !sourceRelative.startsWith("..") && !isAbsolute(sourceRelative);
+			}),
+		).toBe(true);
+
+		const auditCalls = calls.filter((call) => call.name === TOOL_NAME);
 		expect(auditCalls).toHaveLength(1);
 		const auditCall = auditCalls[0];
 		expect(auditCall?.status).toBe("ok");

packages/evals/evals/extensions.docs.eval.ts

@@ -16,7 +16,6 @@ const harness = createPiDocumentationEvalHarness({
 		return {
 			response,
 			extensionErrors: extensions.errors,
-			toolLoaded: extensions.extensions.some(({ tools }) => tools.has(TOOL_NAME)),
 			toolResult: result?.role === "toolResult" ? contentText(result.content) : null,
 		};
 	},
@@ -31,7 +30,6 @@ describeEval(
 				expected: {
 					response: TOOL_RESULT,
 					extensionErrors: [],
-					toolLoaded: true,
 					toolResult: TOOL_RESULT,
 				},
 				match: "strict",

packages/evals/evals/prompt-templates.docs.eval.ts

@@ -0,0 +1,79 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { contentText } from "@earendil-works/pi-ai";
+import { describeEval, StructuredOutputJudge } from "vitest-evals";
+import { createPiDocumentationEvalHarness } from "../src/harness.ts";
+
+const TEMPLATE_NAME = "current-time";
+const COMMAND = `/${TEMPLATE_NAME}`;
+const TEMPLATE_PROMPT = "Tell me the current time and timezone.";
+
+const SCENARIOS = [
+	{
+		title: "Create a project prompt template",
+		request: `This project is already trusted. Add a ${COMMAND} prompt template for this project containing exactly: ${TEMPLATE_PROMPT}`,
+		scope: "project",
+	},
+	{
+		title: "Create a user prompt template",
+		request: `Make a ${COMMAND} prompt template available in all my Pi sessions containing exactly: ${TEMPLATE_PROMPT}`,
+		scope: "user",
+	},
+] as const;
+
+function registerScenario(scenario: (typeof SCENARIOS)[number]): void {
+	const harness = createPiDocumentationEvalHarness({
+		output: ({ session, agentDir }) => {
+			const workspace = session.sessionManager.getCwd();
+			const projectPath = join(workspace, ".pi", "prompts", `${TEMPLATE_NAME}.md`);
+			const userPath = join(agentDir, "prompts", `${TEMPLATE_NAME}.md`);
+			const expectedPath = scenario.scope === "project" ? projectPath : userPath;
+			const otherPath = scenario.scope === "project" ? userPath : projectPath;
+			const { prompts, diagnostics } = session.resourceLoader.getPrompts();
+			const extensions = session.resourceLoader.getExtensions();
+			const matchingTemplates = prompts.filter(({ name }) => name === TEMPLATE_NAME);
+			const template = matchingTemplates.length === 1 ? matchingTemplates[0] : undefined;
+			const expandedPrompt = [...session.messages]
+				.reverse()
+				.find((message) => message.role === "user");
+			const expandedText = expandedPrompt?.role === "user" ? contentText(expandedPrompt.content).trim() : "";
+
+			return {
+				template: template
+					? { scope: template.sourceInfo.scope, atExpectedPath: template.filePath === expectedPath }
+					: null,
+				promptDiagnostics: diagnostics.map(({ message }) => message),
+				extensions: extensions.extensions.filter(({ hidden }) => !hidden).map(({ path }) => path),
+				extensionErrors: extensions.errors.map(({ error }) => error),
+				skills: session.resourceLoader.getSkills().skills.map(({ name }) => name),
+				otherScopeTemplateExists: existsSync(otherPath),
+				expandedToTimeRequest: expandedText === TEMPLATE_PROMPT,
+			};
+		},
+	});
+	const judge = StructuredOutputJudge({
+		expected: {
+			template: { scope: scenario.scope, atExpectedPath: true },
+			promptDiagnostics: [],
+			extensions: [],
+			extensionErrors: [],
+			skills: [],
+			otherScopeTemplateExists: false,
+			expandedToTimeRequest: true,
+		},
+		match: "strict",
+		allowExtras: false,
+	});
+
+	describeEval(scenario.title, { harness, judges: [judge], judgeThreshold: null }, (it) => {
+		it(`creates, reloads, and invokes ${COMMAND}`, async ({ run }) => {
+			await run([
+				{ type: "prompt", content: scenario.request },
+				{ type: "reload" },
+				{ type: "prompt", content: COMMAND },
+			]);
+		});
+	});
+}
+
+for (const scenario of SCENARIOS) registerScenario(scenario);

packages/evals/package.json

@@ -7,6 +7,7 @@
 		"clean": "shx rm -rf .eval",
 		"eval": "npm run eval:host && npm run eval:docs --",
 		"eval:host": "vitest run --config vitest.evals.config.ts --project host",
+		"eval:audit": "vitest run --config vitest.evals.config.ts --project audit",
 		"eval:docs": "node --experimental-strip-types src/cli.ts",
 		"test": "vitest run --config vitest.test.config.ts"
 	},

packages/evals/src/report.ts

@@ -98,7 +98,7 @@ function validateScore(value: unknown): number | undefined {
 	return value;
 }
 
-export function classifyCaseStatus(status: ReportCase["status"]): "errored" | "skipped" | "pending" | undefined {
+function classifyCaseStatus(status: ReportCase["status"]): "errored" | "skipped" | "pending" | undefined {
 	if (status === "failed") return "errored";
 	if (status === "skipped" || status === "todo" || status === "disabled") return "skipped";
 	if (status === "pending") return "pending";
@@ -145,15 +145,16 @@ export async function readTaskObservation(
 	if (!loaded) return { ...identity, outcome: "errored" };
 	const [{ workspace }, rawReport] = loaded;
 	const reportedFullName = `${task.evalSet} ${task.caseId}`;
-	const assertions = rawReport.testResults.flatMap(({ assertionResults }) => assertionResults);
+	const assertions = rawReport.testResults
+		.flatMap(({ assertionResults }) => assertionResults)
+		.filter(({ fullName }) => fullName === reportedFullName);
 	if (assertions.length !== 1) return { ...identity, outcome: "errored" };
 	const assertion = assertions[0];
-	if (assertion.fullName !== reportedFullName) return { ...identity, outcome: "errored" };
 	const statusOutcome = classifyCaseStatus(assertion.status);
 	if (statusOutcome === "skipped" || statusOutcome === "pending") return { ...identity, outcome: statusOutcome };
-	if (workspace.cases.length !== 1) return { ...identity, outcome: "errored" };
-	const caseResult = workspace.cases[0];
-	if (caseResult.fullName !== reportedFullName) return { ...identity, outcome: "errored" };
+	const caseResults = workspace.cases.filter(({ fullName }) => fullName === reportedFullName);
+	if (caseResults.length !== 1) return { ...identity, outcome: "errored" };
+	const caseResult = caseResults[0];
 	if (caseResult.status !== assertion.status) return { ...identity, outcome: "errored" };
 	const run = caseResult.harness?.run;
 	if (!run) return { ...identity, outcome: "errored" };

packages/evals/test/comparison.test.ts

@@ -78,6 +78,8 @@ describe("summarizeEvalObservations", () => {
 				lift: 0.5,
 				totalTokens: { eligiblePairs: 2, controlMean: 150, treatmentMean: 150, meanDelta: 0 },
 				toolCalls: { eligiblePairs: 2, controlMean: 2.5, treatmentMean: 2, meanDelta: -0.5 },
+				totalMs: { eligiblePairs: 2, controlMean: 1000, treatmentMean: 900, meanDelta: -100 },
+				estimatedCostUsd: { eligiblePairs: 2, controlMean: 0.01, treatmentMean: 0.01, meanDelta: 0 },
 			}),
 		]);
 	});

packages/evals/test/configured-runtime.test.ts

@@ -1,6 +1,7 @@
 import { mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
+import { ModelRuntime } from "@earendil-works/pi-coding-agent";
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
 import {
 	createAcmeServer,
@@ -9,7 +10,7 @@ import {
 	OPENAI_PROBE_RESPONSE,
 	OPENAI_PROVIDER_ID,
 } from "../evals/acme-server.ts";
-import { inspectAddedModel, inspectProvider, loadConfiguredModelRuntime } from "../evals/configured-runtime.ts";
+import { inspectAddedModel, inspectProvider } from "../evals/configured-runtime.ts";
 
 const PROVIDER_ID = OPENAI_PROVIDER_ID;
 const MODEL = {
@@ -43,6 +44,15 @@ async function runtimeWith(modelsJson: unknown) {
 	return loadConfiguredModelRuntime(await agentDirWith(modelsJson));
 }
 
+function loadConfiguredModelRuntime(agentDir: string): Promise<ModelRuntime> {
+	return ModelRuntime.create({
+		modelsPath: join(agentDir, "models.json"),
+		authPath: join(agentDir, "auth.json"),
+		modelsStorePath: join(agentDir, "models-store.json"),
+		allowModelNetwork: false,
+	});
+}
+
 function acmeModelsJson() {
 	return {
 		providers: {

packages/evals/test/report.test.ts

@@ -3,7 +3,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterEach, describe, expect, it } from "vitest";
 import type { EvalTask } from "../src/plan.ts";
-import { classifyCaseStatus, PI_SESSION_SNAPSHOT_ARTIFACT, readTaskObservation } from "../src/report.ts";
+import { PI_SESSION_SNAPSHOT_ARTIFACT, readTaskObservation } from "../src/report.ts";
 
 const temporaryDirectories: string[] = [];
 afterEach(async () => {
@@ -22,40 +22,41 @@ const task: EvalTask = {
 const SESSION = '{"type":"session"}\n';
 
 type ReportAssertion = {
-	status?: "passed" | "skipped" | "pending" | "failed";
+	caseId?: string;
+	status?: "passed" | "skipped" | "pending" | "todo" | "disabled" | "failed";
 	meta?: Record<string, unknown>;
 };
 
-async function writeTaskReport(assertion: ReportAssertion = {}) {
+async function writeTaskReport(assertions: ReportAssertion[]) {
 	const directory = await mkdtemp(join(tmpdir(), "pi-eval-report-test-"));
 	temporaryDirectories.push(directory);
 	const reportPath = join(directory, "vitest.json");
-	const status = assertion.status ?? "passed";
+	const results = assertions.map(({ caseId = task.caseId, status = "passed", meta = {} }) => ({
+		ancestorTitles: [task.evalSet],
+		fullName: `${task.evalSet} ${caseId}`,
+		status,
+		title: caseId,
+		failureMessages: [],
+		meta,
+	}));
+	const count = (status: ReportAssertion["status"]): number =>
+		results.filter((result) => result.status === status).length;
 	await writeFile(
 		reportPath,
 		JSON.stringify({
-			numFailedTests: 0,
-			numPassedTests: status === "passed" ? 1 : 0,
-			numPendingTests: status === "pending" || status === "skipped" ? 1 : 0,
-			numTodoTests: 0,
-			numTotalTests: 1,
+			numFailedTests: count("failed"),
+			numPassedTests: count("passed"),
+			numPendingTests: count("pending") + count("skipped") + count("disabled"),
+			numTodoTests: count("todo"),
+			numTotalTests: results.length,
 			startTime: 0,
-			success: true,
+			success: count("failed") === 0,
 			testResults: [
 				{
 					message: "",
 					name: "/repo/packages/evals/evals/example.docs.eval.ts",
-					status: "passed",
-					assertionResults: [
-						{
-							ancestorTitles: [task.evalSet],
-							fullName: `${task.evalSet} ${task.caseId}`,
-							status,
-							title: task.caseId,
-							failureMessages: [],
-							meta: assertion.meta ?? {},
-						},
-					],
+					status: count("failed") === 0 ? "passed" : "failed",
+					assertionResults: results,
 				},
 			],
 		}),
@@ -97,26 +98,19 @@ function scoredMeta(overrides?: {
 	};
 }
 
-async function readObservation(assertion?: ReportAssertion) {
-	const { directory, reportPath } = await writeTaskReport(assertion);
+async function readObservation(assertion: ReportAssertion = {}) {
+	const { directory, reportPath } = await writeTaskReport([assertion]);
 	return { directory, observation: await readTaskObservation(task, reportPath, directory) };
 }
 
-describe("classifyCaseStatus", () => {
-	it.each(["skipped", "todo", "disabled"] as const)("maps %s to skipped", (status) => {
-		expect(classifyCaseStatus(status)).toBe("skipped");
-	});
-
-	it("maps failed infrastructure to errored", () => {
-		expect(classifyCaseStatus("failed")).toBe("errored");
-	});
-});
-
 describe("readTaskObservation", () => {
-	it("preserves a skipped outcome when no harness run exists", async () => {
-		const { observation } = await readObservation({ status: "skipped" });
-		expect(observation).toMatchObject({ outcome: "skipped" });
-	});
+	it.each(["skipped", "todo", "disabled"] as const)(
+		"preserves a skipped outcome for %s cases without harness runs",
+		async (status) => {
+			const { observation } = await readObservation({ status });
+			expect(observation).toMatchObject({ outcome: "skipped" });
+		},
+	);
 
 	it("preserves a pending outcome when no harness run exists", async () => {
 		const { observation } = await readObservation({ status: "pending" });
@@ -126,7 +120,7 @@ describe("readTaskObservation", () => {
 	it("preserves metrics from a failed eval with a partial harness run", async () => {
 		const { directory, observation } = await readObservation({
 			status: "failed",
-			meta: scoredMeta({ errors: [{ message: "Prompt verification failed" }] }),
+			meta: scoredMeta(),
 		});
 		expect(observation).toEqual({
 			evalSet: task.evalSet,
@@ -181,6 +175,15 @@ describe("readTaskObservation", () => {
 		).resolves.toBe(SESSION);
 	});
 
+	it("selects the requested case when a sibling case is skipped", async () => {
+		const { directory, reportPath } = await writeTaskReport([
+			{ caseId: "sibling case", status: "skipped" },
+			{ caseId: task.caseId, meta: scoredMeta() },
+		]);
+		const observation = await readTaskObservation(task, reportPath, directory);
+		expect(observation).toMatchObject({ outcome: "scored", score: 0.5, totalTokens: 15 });
+	});
+
 	it("treats a zero score as scored data", async () => {
 		const { observation } = await readObservation({ meta: scoredMeta({ avgScore: 0 }) });
 		expect(observation).toMatchObject({ outcome: "scored", score: 0 });

packages/evals/vitest.evals.config.ts

@@ -24,7 +24,17 @@ const evalConfig = defineConfig({
 					test: {
 						name: "host",
 						include: ["evals/**/*.eval.ts"],
-						exclude: ["evals/**/*.docs.eval.ts"],
+						exclude: ["evals/**/*.docs.eval.ts", "evals/documentation-audit.eval.ts"],
+						sequence: { concurrent: false },
+						testTimeout: 300_000,
+						hookTimeout: 300_000,
+					},
+				},
+				{
+					extends: true,
+					test: {
+						name: "audit",
+						include: ["evals/documentation-audit.eval.ts"],
 						sequence: { concurrent: false },
 						testTimeout: 300_000,
 						hookTimeout: 300_000,