diff --git a/packages/evals/README.md b/packages/evals/README.md index 74c4904b0..406dec988 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -7,7 +7,7 @@ Behavioral evals for Pi's coding agent, built with `vitest-evals`. Eval definitions are flat under `evals/`: - `*.docs.eval.ts` is a documentation-lift eval. `eval:docs` runs each case in isolated `without_docs` and `with_docs` containers and reports lift. -- Other `*.eval.ts` files are host evals. `eval:host` runs them with Vitest on this machine. They are ordinary vitest-evals suites, not paired comparisons. +- Other `*.eval.ts` files are host evals. `eval:host` runs the smoke suite; `eval:audit` runs the documentation audit. They are ordinary vitest-evals suites, not paired comparisons. Runner code lives in `src/`: @@ -21,25 +21,25 @@ Eval suites and their fixtures live under `evals/`. Image build files live in `d ## Run evals -Host evals (smoke, documentation audit) and documentation-lift evals need `PI_PROVIDER` and `PI_MODEL`. +The host smoke eval, on-demand documentation audit, and documentation-lift evals need `PI_PROVIDER` and `PI_MODEL`. ```bash PI_PROVIDER=openai-codex PI_MODEL=gpt-5.6-sol npm run eval -w packages/evals ``` -That runs host evals, then the documentation comparison. Extra CLI flags after `--` go to `eval:docs` only. +That runs the host smoke eval, then the documentation comparison. The documentation audit remains explicit because it scans every documentation page. Extra CLI flags after `--` go to `eval:docs` only. -Host only: +Host smoke only: ```bash PI_PROVIDER=openai-codex PI_MODEL=gpt-5.6-sol npm run eval:host -w packages/evals ``` -One host suite: +Documentation audit: ```bash PI_PROVIDER=openai-codex PI_MODEL=gpt-5.6-sol \ - npm run eval:host -w packages/evals -- evals/documentation-audit.eval.ts + npm run eval:audit -w packages/evals ``` ## Run documentation comparisons diff --git a/packages/evals/evals/configured-runtime.ts b/packages/evals/evals/configured-runtime.ts index 8aaca0eee..c343634d5 100644 --- a/packages/evals/evals/configured-runtime.ts +++ b/packages/evals/evals/configured-runtime.ts @@ -1,4 +1,3 @@ -import { join } from "node:path"; import { type Api, type Context, contentText, type Model, type ModelsSimpleStreamOptions } from "@earendil-works/pi-ai"; import { ModelRuntime } from "@earendil-works/pi-coding-agent"; @@ -53,15 +52,6 @@ function modelFields(model: Model): ModelFields { }; } -export function loadConfiguredModelRuntime(agentDir: string): Promise { - return ModelRuntime.create({ - modelsPath: join(agentDir, "models.json"), - authPath: join(agentDir, "auth.json"), - modelsStorePath: join(agentDir, "models-store.json"), - allowModelNetwork: false, - }); -} - export async function inspectProvider( runtime: ModelRuntime, scenario: ProviderScenario, diff --git a/packages/evals/evals/documentation-audit.eval.ts b/packages/evals/evals/documentation-audit.eval.ts index dc2d6a582..baf8c6f0f 100644 --- a/packages/evals/evals/documentation-audit.eval.ts +++ b/packages/evals/evals/documentation-audit.eval.ts @@ -1,5 +1,5 @@ import { globSync } from "node:fs"; -import { resolve } from "node:path"; +import { isAbsolute, relative, resolve } from "node:path"; import { Type } from "@earendil-works/pi-ai"; import { defineTool } from "@earendil-works/pi-coding-agent"; import { expect } from "vitest"; @@ -31,6 +31,7 @@ const submitAudit = defineTool({ const repositoryRoot = resolve(import.meta.dirname, "../../.."); const docsRoot = resolve(repositoryRoot, "packages/coding-agent/docs"); +const sourceRoot = resolve(repositoryRoot, "packages/coding-agent/src"); const pages = globSync("**/*.md", { cwd: docsRoot }) .map((path) => ({ path: path.replaceAll("\\", "/") })) .sort((left, right) => left.path.localeCompare(right.path)); @@ -56,7 +57,20 @@ For a mismatch, quote the claim, cite the implementation path and symbol, and st Call ${TOOL_NAME} exactly once as your final action. Do not return prose.`); - const auditCalls = toolCalls(result).filter((call) => call.name === TOOL_NAME); + const calls = toolCalls(result); + const readPaths = calls.flatMap((call) => { + const path = call.name === "read" && call.status === "ok" ? call.arguments?.path : undefined; + return typeof path === "string" ? [resolve(path)] : []; + }); + expect(readPaths).toContain(documentationPath); + expect( + readPaths.some((readPath) => { + const sourceRelative = relative(sourceRoot, readPath); + return sourceRelative !== "" && !sourceRelative.startsWith("..") && !isAbsolute(sourceRelative); + }), + ).toBe(true); + + const auditCalls = calls.filter((call) => call.name === TOOL_NAME); expect(auditCalls).toHaveLength(1); const auditCall = auditCalls[0]; expect(auditCall?.status).toBe("ok"); diff --git a/packages/evals/evals/extensions.docs.eval.ts b/packages/evals/evals/extensions.docs.eval.ts index ef21c8f60..5fd51666d 100644 --- a/packages/evals/evals/extensions.docs.eval.ts +++ b/packages/evals/evals/extensions.docs.eval.ts @@ -16,7 +16,6 @@ const harness = createPiDocumentationEvalHarness({ return { response, extensionErrors: extensions.errors, - toolLoaded: extensions.extensions.some(({ tools }) => tools.has(TOOL_NAME)), toolResult: result?.role === "toolResult" ? contentText(result.content) : null, }; }, @@ -31,7 +30,6 @@ describeEval( expected: { response: TOOL_RESULT, extensionErrors: [], - toolLoaded: true, toolResult: TOOL_RESULT, }, match: "strict", diff --git a/packages/evals/evals/prompt-templates.docs.eval.ts b/packages/evals/evals/prompt-templates.docs.eval.ts new file mode 100644 index 000000000..08edd2238 --- /dev/null +++ b/packages/evals/evals/prompt-templates.docs.eval.ts @@ -0,0 +1,79 @@ +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { contentText } from "@earendil-works/pi-ai"; +import { describeEval, StructuredOutputJudge } from "vitest-evals"; +import { createPiDocumentationEvalHarness } from "../src/harness.ts"; + +const TEMPLATE_NAME = "current-time"; +const COMMAND = `/${TEMPLATE_NAME}`; +const TEMPLATE_PROMPT = "Tell me the current time and timezone."; + +const SCENARIOS = [ + { + title: "Create a project prompt template", + request: `This project is already trusted. Add a ${COMMAND} prompt template for this project containing exactly: ${TEMPLATE_PROMPT}`, + scope: "project", + }, + { + title: "Create a user prompt template", + request: `Make a ${COMMAND} prompt template available in all my Pi sessions containing exactly: ${TEMPLATE_PROMPT}`, + scope: "user", + }, +] as const; + +function registerScenario(scenario: (typeof SCENARIOS)[number]): void { + const harness = createPiDocumentationEvalHarness({ + output: ({ session, agentDir }) => { + const workspace = session.sessionManager.getCwd(); + const projectPath = join(workspace, ".pi", "prompts", `${TEMPLATE_NAME}.md`); + const userPath = join(agentDir, "prompts", `${TEMPLATE_NAME}.md`); + const expectedPath = scenario.scope === "project" ? projectPath : userPath; + const otherPath = scenario.scope === "project" ? userPath : projectPath; + const { prompts, diagnostics } = session.resourceLoader.getPrompts(); + const extensions = session.resourceLoader.getExtensions(); + const matchingTemplates = prompts.filter(({ name }) => name === TEMPLATE_NAME); + const template = matchingTemplates.length === 1 ? matchingTemplates[0] : undefined; + const expandedPrompt = [...session.messages] + .reverse() + .find((message) => message.role === "user"); + const expandedText = expandedPrompt?.role === "user" ? contentText(expandedPrompt.content).trim() : ""; + + return { + template: template + ? { scope: template.sourceInfo.scope, atExpectedPath: template.filePath === expectedPath } + : null, + promptDiagnostics: diagnostics.map(({ message }) => message), + extensions: extensions.extensions.filter(({ hidden }) => !hidden).map(({ path }) => path), + extensionErrors: extensions.errors.map(({ error }) => error), + skills: session.resourceLoader.getSkills().skills.map(({ name }) => name), + otherScopeTemplateExists: existsSync(otherPath), + expandedToTimeRequest: expandedText === TEMPLATE_PROMPT, + }; + }, + }); + const judge = StructuredOutputJudge({ + expected: { + template: { scope: scenario.scope, atExpectedPath: true }, + promptDiagnostics: [], + extensions: [], + extensionErrors: [], + skills: [], + otherScopeTemplateExists: false, + expandedToTimeRequest: true, + }, + match: "strict", + allowExtras: false, + }); + + describeEval(scenario.title, { harness, judges: [judge], judgeThreshold: null }, (it) => { + it(`creates, reloads, and invokes ${COMMAND}`, async ({ run }) => { + await run([ + { type: "prompt", content: scenario.request }, + { type: "reload" }, + { type: "prompt", content: COMMAND }, + ]); + }); + }); +} + +for (const scenario of SCENARIOS) registerScenario(scenario); diff --git a/packages/evals/package.json b/packages/evals/package.json index b36546214..e7cc7bde7 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -7,6 +7,7 @@ "clean": "shx rm -rf .eval", "eval": "npm run eval:host && npm run eval:docs --", "eval:host": "vitest run --config vitest.evals.config.ts --project host", + "eval:audit": "vitest run --config vitest.evals.config.ts --project audit", "eval:docs": "node --experimental-strip-types src/cli.ts", "test": "vitest run --config vitest.test.config.ts" }, diff --git a/packages/evals/src/report.ts b/packages/evals/src/report.ts index 7c5e1d093..cd7192d39 100644 --- a/packages/evals/src/report.ts +++ b/packages/evals/src/report.ts @@ -98,7 +98,7 @@ function validateScore(value: unknown): number | undefined { return value; } -export function classifyCaseStatus(status: ReportCase["status"]): "errored" | "skipped" | "pending" | undefined { +function classifyCaseStatus(status: ReportCase["status"]): "errored" | "skipped" | "pending" | undefined { if (status === "failed") return "errored"; if (status === "skipped" || status === "todo" || status === "disabled") return "skipped"; if (status === "pending") return "pending"; @@ -145,15 +145,16 @@ export async function readTaskObservation( if (!loaded) return { ...identity, outcome: "errored" }; const [{ workspace }, rawReport] = loaded; const reportedFullName = `${task.evalSet} ${task.caseId}`; - const assertions = rawReport.testResults.flatMap(({ assertionResults }) => assertionResults); + const assertions = rawReport.testResults + .flatMap(({ assertionResults }) => assertionResults) + .filter(({ fullName }) => fullName === reportedFullName); if (assertions.length !== 1) return { ...identity, outcome: "errored" }; const assertion = assertions[0]; - if (assertion.fullName !== reportedFullName) return { ...identity, outcome: "errored" }; const statusOutcome = classifyCaseStatus(assertion.status); if (statusOutcome === "skipped" || statusOutcome === "pending") return { ...identity, outcome: statusOutcome }; - if (workspace.cases.length !== 1) return { ...identity, outcome: "errored" }; - const caseResult = workspace.cases[0]; - if (caseResult.fullName !== reportedFullName) return { ...identity, outcome: "errored" }; + const caseResults = workspace.cases.filter(({ fullName }) => fullName === reportedFullName); + if (caseResults.length !== 1) return { ...identity, outcome: "errored" }; + const caseResult = caseResults[0]; if (caseResult.status !== assertion.status) return { ...identity, outcome: "errored" }; const run = caseResult.harness?.run; if (!run) return { ...identity, outcome: "errored" }; diff --git a/packages/evals/test/comparison.test.ts b/packages/evals/test/comparison.test.ts index ba2ad550d..4a00bac8e 100644 --- a/packages/evals/test/comparison.test.ts +++ b/packages/evals/test/comparison.test.ts @@ -78,6 +78,8 @@ describe("summarizeEvalObservations", () => { lift: 0.5, totalTokens: { eligiblePairs: 2, controlMean: 150, treatmentMean: 150, meanDelta: 0 }, toolCalls: { eligiblePairs: 2, controlMean: 2.5, treatmentMean: 2, meanDelta: -0.5 }, + totalMs: { eligiblePairs: 2, controlMean: 1000, treatmentMean: 900, meanDelta: -100 }, + estimatedCostUsd: { eligiblePairs: 2, controlMean: 0.01, treatmentMean: 0.01, meanDelta: 0 }, }), ]); }); diff --git a/packages/evals/test/configured-runtime.test.ts b/packages/evals/test/configured-runtime.test.ts index 7b74c94d3..267151486 100644 --- a/packages/evals/test/configured-runtime.test.ts +++ b/packages/evals/test/configured-runtime.test.ts @@ -1,6 +1,7 @@ import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { ModelRuntime } from "@earendil-works/pi-coding-agent"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { createAcmeServer, @@ -9,7 +10,7 @@ import { OPENAI_PROBE_RESPONSE, OPENAI_PROVIDER_ID, } from "../evals/acme-server.ts"; -import { inspectAddedModel, inspectProvider, loadConfiguredModelRuntime } from "../evals/configured-runtime.ts"; +import { inspectAddedModel, inspectProvider } from "../evals/configured-runtime.ts"; const PROVIDER_ID = OPENAI_PROVIDER_ID; const MODEL = { @@ -43,6 +44,15 @@ async function runtimeWith(modelsJson: unknown) { return loadConfiguredModelRuntime(await agentDirWith(modelsJson)); } +function loadConfiguredModelRuntime(agentDir: string): Promise { + return ModelRuntime.create({ + modelsPath: join(agentDir, "models.json"), + authPath: join(agentDir, "auth.json"), + modelsStorePath: join(agentDir, "models-store.json"), + allowModelNetwork: false, + }); +} + function acmeModelsJson() { return { providers: { diff --git a/packages/evals/test/report.test.ts b/packages/evals/test/report.test.ts index f872ea4f3..594f2b96a 100644 --- a/packages/evals/test/report.test.ts +++ b/packages/evals/test/report.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import type { EvalTask } from "../src/plan.ts"; -import { classifyCaseStatus, PI_SESSION_SNAPSHOT_ARTIFACT, readTaskObservation } from "../src/report.ts"; +import { PI_SESSION_SNAPSHOT_ARTIFACT, readTaskObservation } from "../src/report.ts"; const temporaryDirectories: string[] = []; afterEach(async () => { @@ -22,40 +22,41 @@ const task: EvalTask = { const SESSION = '{"type":"session"}\n'; type ReportAssertion = { - status?: "passed" | "skipped" | "pending" | "failed"; + caseId?: string; + status?: "passed" | "skipped" | "pending" | "todo" | "disabled" | "failed"; meta?: Record; }; -async function writeTaskReport(assertion: ReportAssertion = {}) { +async function writeTaskReport(assertions: ReportAssertion[]) { const directory = await mkdtemp(join(tmpdir(), "pi-eval-report-test-")); temporaryDirectories.push(directory); const reportPath = join(directory, "vitest.json"); - const status = assertion.status ?? "passed"; + const results = assertions.map(({ caseId = task.caseId, status = "passed", meta = {} }) => ({ + ancestorTitles: [task.evalSet], + fullName: `${task.evalSet} ${caseId}`, + status, + title: caseId, + failureMessages: [], + meta, + })); + const count = (status: ReportAssertion["status"]): number => + results.filter((result) => result.status === status).length; await writeFile( reportPath, JSON.stringify({ - numFailedTests: 0, - numPassedTests: status === "passed" ? 1 : 0, - numPendingTests: status === "pending" || status === "skipped" ? 1 : 0, - numTodoTests: 0, - numTotalTests: 1, + numFailedTests: count("failed"), + numPassedTests: count("passed"), + numPendingTests: count("pending") + count("skipped") + count("disabled"), + numTodoTests: count("todo"), + numTotalTests: results.length, startTime: 0, - success: true, + success: count("failed") === 0, testResults: [ { message: "", name: "/repo/packages/evals/evals/example.docs.eval.ts", - status: "passed", - assertionResults: [ - { - ancestorTitles: [task.evalSet], - fullName: `${task.evalSet} ${task.caseId}`, - status, - title: task.caseId, - failureMessages: [], - meta: assertion.meta ?? {}, - }, - ], + status: count("failed") === 0 ? "passed" : "failed", + assertionResults: results, }, ], }), @@ -97,26 +98,19 @@ function scoredMeta(overrides?: { }; } -async function readObservation(assertion?: ReportAssertion) { - const { directory, reportPath } = await writeTaskReport(assertion); +async function readObservation(assertion: ReportAssertion = {}) { + const { directory, reportPath } = await writeTaskReport([assertion]); return { directory, observation: await readTaskObservation(task, reportPath, directory) }; } -describe("classifyCaseStatus", () => { - it.each(["skipped", "todo", "disabled"] as const)("maps %s to skipped", (status) => { - expect(classifyCaseStatus(status)).toBe("skipped"); - }); - - it("maps failed infrastructure to errored", () => { - expect(classifyCaseStatus("failed")).toBe("errored"); - }); -}); - describe("readTaskObservation", () => { - it("preserves a skipped outcome when no harness run exists", async () => { - const { observation } = await readObservation({ status: "skipped" }); - expect(observation).toMatchObject({ outcome: "skipped" }); - }); + it.each(["skipped", "todo", "disabled"] as const)( + "preserves a skipped outcome for %s cases without harness runs", + async (status) => { + const { observation } = await readObservation({ status }); + expect(observation).toMatchObject({ outcome: "skipped" }); + }, + ); it("preserves a pending outcome when no harness run exists", async () => { const { observation } = await readObservation({ status: "pending" }); @@ -126,7 +120,7 @@ describe("readTaskObservation", () => { it("preserves metrics from a failed eval with a partial harness run", async () => { const { directory, observation } = await readObservation({ status: "failed", - meta: scoredMeta({ errors: [{ message: "Prompt verification failed" }] }), + meta: scoredMeta(), }); expect(observation).toEqual({ evalSet: task.evalSet, @@ -181,6 +175,15 @@ describe("readTaskObservation", () => { ).resolves.toBe(SESSION); }); + it("selects the requested case when a sibling case is skipped", async () => { + const { directory, reportPath } = await writeTaskReport([ + { caseId: "sibling case", status: "skipped" }, + { caseId: task.caseId, meta: scoredMeta() }, + ]); + const observation = await readTaskObservation(task, reportPath, directory); + expect(observation).toMatchObject({ outcome: "scored", score: 0.5, totalTokens: 15 }); + }); + it("treats a zero score as scored data", async () => { const { observation } = await readObservation({ meta: scoredMeta({ avgScore: 0 }) }); expect(observation).toMatchObject({ outcome: "scored", score: 0 }); diff --git a/packages/evals/vitest.evals.config.ts b/packages/evals/vitest.evals.config.ts index ced768eaf..20ecd73e7 100644 --- a/packages/evals/vitest.evals.config.ts +++ b/packages/evals/vitest.evals.config.ts @@ -24,7 +24,17 @@ const evalConfig = defineConfig({ test: { name: "host", include: ["evals/**/*.eval.ts"], - exclude: ["evals/**/*.docs.eval.ts"], + exclude: ["evals/**/*.docs.eval.ts", "evals/documentation-audit.eval.ts"], + sequence: { concurrent: false }, + testTimeout: 300_000, + hookTimeout: 300_000, + }, + }, + { + extends: true, + test: { + name: "audit", + include: ["evals/documentation-audit.eval.ts"], sequence: { concurrent: false }, testTimeout: 300_000, hookTimeout: 300_000,