diff --git a/packages/coding-agent/src/core/extensions/loader.ts b/packages/coding-agent/src/core/extensions/loader.ts index 89f59c03a..3e66f390b 100644 --- a/packages/coding-agent/src/core/extensions/loader.ts +++ b/packages/coding-agent/src/core/extensions/loader.ts @@ -73,8 +73,10 @@ const VIRTUAL_MODULES: Record = { const require = createRequire(import.meta.url); +const isTypeScriptSourceRuntime = !isBunBinary && path.extname(fileURLToPath(import.meta.url)) === ".ts"; + /** - * Get aliases for jiti (used in Node.js/development mode). + * Get aliases for jiti (used in built Node.js mode). * In Bun binary mode, virtualModules is used instead. */ let _aliases: Record | null = null; @@ -410,10 +412,13 @@ async function loadExtensionModule(extensionPath: string, cacheToken?: Extension const jiti = createJiti(import.meta.url, { moduleCache: false, - // In Bun binary: use virtualModules for bundled packages (no filesystem resolution) - // Also disable tryNative so jiti handles ALL imports (not just the entry point) - // In Node.js/dev: use aliases to resolve to node_modules paths - ...(isBunBinary ? { virtualModules: VIRTUAL_MODULES, tryNative: false } : { alias: getAliases() }), + // Bun uses modules embedded in the executable. Source TypeScript reuses the + // host-resolved modules and root tsconfig paths. Built Node uses dist aliases. + ...(isBunBinary + ? { virtualModules: VIRTUAL_MODULES, tryNative: false } + : isTypeScriptSourceRuntime + ? { virtualModules: VIRTUAL_MODULES, tsconfigPaths: true } + : { alias: getAliases() }), }); const module = await jiti.import(extensionPath, { default: true }); diff --git a/packages/evals/.gitignore b/packages/evals/.gitignore new file mode 100644 index 000000000..3270c5ed7 --- /dev/null +++ b/packages/evals/.gitignore @@ -0,0 +1 @@ +.eval/ diff --git a/packages/evals/README.md b/packages/evals/README.md index dabb0fa10..b76d40d29 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -1,32 +1,153 @@ # Pi evals -Behavioral evaluations for Pi using `vitest-evals`. +Pi evals are behavioral, model-backed checks for Pi workflows. They adapt a real `AgentSession` to `vitest-evals`, run +it in isolated temporary project and agent directories, and attach native Pi session artifacts. +Use them to measure end-to-end behavior and compare prompts, tools, skills, models, or other harness configurations. -`src/pi-harness.ts` adapts Pi's `AgentSession` directly to `vitest-evals`. Each run uses isolated temporary project and -agent directories that are removed afterward. Judge harnesses are configured separately from the application harness. +## Running evals -Eval cases supply either one prompt or a JSON-safe sequence of prompt and reload steps. The harness executes that input -and normalizes the resulting session without defining scenario-specific behavior. Suites can configure a typed output -selector over the final response and `AgentSession` state. - -## Running - -From the repository root, run with an explicit provider and model: +Run from the repository root with a default provider and model: ```bash -npm run eval -- --provider openai-codex --model gpt-5.4 +npm run eval -- --provider openai --model gpt-5.6-sol ``` -When invoked from a Pi Bash tool, the current session supplies `PI_PROVIDER` and `PI_MODEL`, so this is sufficient: +The equivalent environment variables are: ```bash -npm run eval +PI_PROVIDER=openai PI_MODEL=gpt-5.6-sol npm run eval ``` -The runner requires both values and never falls back to another model. Additional arguments are forwarded to Vitest, for example: +CLI values take precedence and become defaults for harnesses that do not select a model explicitly. Provider and model must be supplied together. The runner also allows no default when every executed harness configures its own model. +Authentication comes from Pi's normal `ModelRuntime`, including Pi subscription credentials and provider API-key +environment variables. + +Additional arguments are forwarded to Vitest: ```bash -npm run eval -- -t "capital of France" +npm run eval -- src/extensions.eval.ts +npm run eval -- -t "creates, reloads, and uses" ``` -Authentication is resolved by Pi's normal `ModelRuntime`. Subscription-backed providers such as `openai-codex` use credentials from the user's Pi configuration. API-backed providers use their standard environment variables, such as `OPENAI_API_KEY` for `openai`. +Each invocation prints an ignored `.eval/` artifact directory. `runs.jsonl` indexes completed harness runs and their +native Pi session JSONL attachments under `sessions/`. These files may contain prompts, responses, source code, and tool +output. + +## Writing evals + +Follow [`vitest-evals`](https://github.com/getsentry/vitest-evals) for general suite, judge, assertion, and normalized +trace guidance. Pi-specific evals use `createPiCodingAgentHarness(...)` from `src/pi-harness.ts`, with one harness bound +to each `describeEval(...)` suite: + +```ts +import { expect } from "vitest"; +import { describeEval } from "vitest-evals"; +import { createPiCodingAgentHarness } from "./pi-harness.ts"; + +const harness = createPiCodingAgentHarness({ noTools: "all" }); + +describeEval("Pi smoke", { harness }, (it) => { + it("answers a factual question", async ({ run }) => { + const result = await run("What is the capital of France? Reply with only the city name."); + expect(result.output).toBe("Paris"); + }); +}); +``` + +### Configuring the Pi harness + +`createPiCodingAgentHarness(...)` accepts: + +- `name`: stable harness identity used by reports and comparisons. +- `model`: optional `{ provider, id }` selection. It overrides the runner's default model. +- `noTools`: Pi's tool-disable configuration. +- `transformSystemPrompt`: transforms the complete default prompt before the eval starts. +- `output`: transforms the final response and `AgentSession` into a JSON-safe domain result. + +An explicitly selected model makes model-comparison harnesses independent of the runner default: + +```ts +const harness = createPiCodingAgentHarness({ + name: "claude-opus-4-6", + model: { provider: "anthropic", id: "claude-opus-4-6" }, +}); +``` + +A run accepts either one prompt or a sequence of prompt and reload steps. Reload steps are useful when the preceding +prompt creates or changes Pi resources: + +```ts +const result = await run([ + { type: "prompt", content: "Create a Pi extension." }, + { type: "reload" }, + { type: "prompt", content: "Use the extension." }, +]); +``` + +### Transforming harness output + +Use `output` to expose scenario-specific, JSON-safe behavior without adding that behavior to the generic Pi adapter: + +```ts +const harness = createPiCodingAgentHarness({ + output: ({ response, session }) => ({ + response, + activeTools: session.getActiveToolNames(), + extensionErrors: session.resourceLoader.getExtensions().errors, + }), +}); +``` + +Assert application behavior on `result.output`. Assert model and tool traces on `result.session`, using +`vitest-evals` helpers such as `toolCalls(...)`. + +### Writing comparative eval sets + +Use `evalHarnessTable(...)` with Vitest's native `describe.for(...)` to run the same inputs against multiple harnesses. +Harnesses may differ by prompt, tools, skills, model, or any other Pi configuration: + +```ts +import { describe } from "vitest"; +import { createJudge, describeEval } from "vitest-evals"; +import { evalHarnessTable } from "./vitest-evals/harness-table.ts"; + +const TargetTaskJudge = createJudge("TargetTaskJudge", ({ output }) => ({ + score: output === "expected result" ? 1 : 0, +})); + +const harnessTable = evalHarnessTable( + "target skill effectiveness", + { + baseline: withoutTargetSkillHarness, + candidate: withTargetSkillHarness, + repetitions: 6, + }, +); + +describe.for(harnessTable)("$name repetition $repetition", ({ harness }) => { + describeEval("target skill effectiveness", { harness, judges: [TargetTaskJudge], judgeThreshold: null }, (it) => { + it("completes the target task", async ({ run }) => { + await run("Complete the target task."); + }); + }); +}); +``` + +Comparative suites should record correctness with deterministic or model-backed judges and set `judgeThreshold: null`. +This keeps a low score as an observation instead of making the Vitest invocation fail. Use hard assertions only for +suite invariants and infrastructure contracts. `expect.soft(...)` still fails the test and is not a scoring mechanism. + +The Pi harness snapshots native session JSONL before deleting its temporary workspace. An eval-only `afterEach` hook +registers that snapshot against the explicit Vitest test task before reporters run. + +Harness names must be stable and unique within an eval set. The grouping key combines repetition with a non-empty string +`input.id` when available, otherwise with a SHA-256 hash of strict canonical JSON input. Use `candidate` for one treatment +or `candidates` for multiple treatments. Each candidate is compared only with the declared baseline. For each matched +input and repetition, the reporter computes pass-rate lift from each run's recorded average judge score, treating a score +of at least `1` as passing. Lift is the candidate pass rate minus the baseline pass rate, in percentage points. Missing +judge scores are reported as incomplete observations. Tokens, latency, and estimated cost remain separate +candidate-minus-baseline paired deltas; missing telemetry remains unavailable. If execution-order randomization becomes +necessary, use Vitest's built-in sequence shuffling. + +See the [`skill-eval-harness`](https://github.com/adewale/skill-eval-harness/) guidance for comparative-eval methodology, +repetition strategy, trustworthy judges, and telemetry interpretation. diff --git a/packages/evals/package.json b/packages/evals/package.json index 801cc2978..3f1d792cb 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -5,7 +5,8 @@ "type": "module", "scripts": { "clean": "shx rm -rf .eval", - "eval": "node scripts/run-evals.mjs" + "eval": "node scripts/run-evals.mjs", + "test": "vitest run --config vitest.test.config.ts" }, "devDependencies": { "@earendil-works/pi-ai": "^0.83.0", diff --git a/packages/evals/scripts/run-evals.mjs b/packages/evals/scripts/run-evals.mjs index 84799e09d..fa41b2390 100644 --- a/packages/evals/scripts/run-evals.mjs +++ b/packages/evals/scripts/run-evals.mjs @@ -1,9 +1,18 @@ +import { randomUUID } from "node:crypto"; +import { mkdirSync } from "node:fs"; import { spawnSync } from "node:child_process"; import { createRequire } from "node:module"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const artifactDirectory = process.env.PI_EVAL_ARTIFACT_DIR + ? resolve(packageRoot, process.env.PI_EVAL_ARTIFACT_DIR) + : resolve( + packageRoot, + ".eval", + `${new Date().toISOString().replaceAll(":", "-")}_${randomUUID()}`, + ); const args = process.argv.slice(2); let provider; let model; @@ -37,37 +46,47 @@ for (let index = 0; index < args.length; index += 1) { vitestArgs.push(arg); } -if (hasCliModelSelection && (!provider || !model)) { - console.error("CLI model selection requires both --provider and --model."); - process.exit(1); -} - -provider ??= process.env.PI_PROVIDER; -model ??= process.env.PI_MODEL; - -if (!provider || !model) { - console.error( - "No eval model selected. Pass --provider and --model, or set PI_PROVIDER and PI_MODEL.", - ); - process.exit(1); +provider = provider?.trim() || undefined; +model = model?.trim() || undefined; +if (hasCliModelSelection) { + if (!provider || !model) { + console.error("CLI model selection requires both --provider and --model."); + process.exit(1); + } +} else { + provider = process.env.PI_PROVIDER?.trim() || undefined; + model = process.env.PI_MODEL?.trim() || undefined; + if (Boolean(provider) !== Boolean(model)) { + console.error("Default model selection requires both PI_PROVIDER and PI_MODEL."); + process.exit(1); + } } const require = createRequire(import.meta.url); const vitestPackagePath = require.resolve("vitest/package.json"); const vitestCliPath = resolve(dirname(vitestPackagePath), "vitest.mjs"); -console.error(`[eval] provider=${provider} model=${model}`); +mkdirSync(artifactDirectory, { recursive: true, mode: 0o700 }); +console.error(`[eval] default-model=${provider && model ? `${provider}/${model}` : "none"}`); +console.error(`[eval] artifacts=${artifactDirectory}`); +const childEnvironment = { + ...process.env, + PI_EVAL_ARTIFACT_DIR: artifactDirectory, +}; +if (provider && model) { + childEnvironment.PI_PROVIDER = provider; + childEnvironment.PI_MODEL = model; +} else { + delete childEnvironment.PI_PROVIDER; + delete childEnvironment.PI_MODEL; +} const result = spawnSync( process.execPath, [vitestCliPath, "run", "--config", "vitest.config.ts", ...vitestArgs], { cwd: packageRoot, stdio: "inherit", - env: { - ...process.env, - PI_PROVIDER: provider, - PI_MODEL: model, - }, + env: childEnvironment, }, ); diff --git a/packages/evals/src/extensions.eval.ts b/packages/evals/src/extensions.eval.ts index e72a844cd..577b998d6 100644 --- a/packages/evals/src/extensions.eval.ts +++ b/packages/evals/src/extensions.eval.ts @@ -1,42 +1,140 @@ -import { expect } from "vitest"; -import { describeEval, toolCalls } from "vitest-evals"; -import { createPiCodingAgentHarness } from "./pi-harness.ts"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect } from "vitest"; +import { createJudge, describeEval } from "vitest-evals"; +import { createPiCodingAgentHarness, type PiCodingAgentInput } from "./pi-harness.ts"; +import { recordEvalSourceArtifact } from "./vitest-evals/artifacts.ts"; +import { evalHarnessTable } from "./vitest-evals/harness-table.ts"; + +type ExtensionAuthoringOutput = { + response: string; + systemPromptHasGuidelines: boolean; + systemPromptHasPiDocs: boolean; + extensionErrors: Array<{ path: string; error: string }>; + loadedExtensions: Array<{ path: string; tools: string[] }>; + extensionSource: string | null; +}; + +function createExtensionAuthoringHarness(name: string, transformSystemPrompt?: (defaultPrompt: string) => string) { + return createPiCodingAgentHarness({ + name, + ...(transformSystemPrompt ? { transformSystemPrompt } : {}), + output: ({ response, session }) => { + const extensions = session.resourceLoader.getExtensions(); + const extensionPath = join(session.sessionManager.getCwd(), ".pi", "extensions", "hello.ts"); + const extensionSource = existsSync(extensionPath) ? readFileSync(extensionPath, "utf8") : null; + return { + response, + systemPromptHasGuidelines: session.systemPrompt.includes("\nGuidelines:\n"), + systemPromptHasPiDocs: session.systemPrompt.includes("\nPi documentation (read only"), + extensionErrors: extensions.errors, + loadedExtensions: extensions.extensions.map(({ path, tools }) => ({ + path, + tools: [...tools.keys()], + })), + extensionSource, + }; + }, + }); +} + +function excludeGuidelinesAndDocumentation(defaultPrompt: string): string { + const guidelinesStart = defaultPrompt.indexOf("\nGuidelines:\n"); + if (guidelinesStart === -1) throw new Error("Default Pi system prompt has no Guidelines section."); + return defaultPrompt.slice(0, guidelinesStart); +} + +function prepareDefaultPromptOverride(defaultPrompt: string): string { + const cwdStart = defaultPrompt.lastIndexOf("\nCurrent working directory: "); + if (cwdStart === -1) throw new Error("Default Pi system prompt has no working-directory section."); + return defaultPrompt.slice(0, cwdStart); +} + +const ExtensionAuthoringJudge = createJudge( + "ExtensionAuthoringJudge", + ({ output, toolCalls }) => { + const failures: string[] = []; + if (output.extensionSource === null) { + failures.push("generated extension source is unavailable"); + } else { + const imports = Array.from( + output.extensionSource.matchAll(/\b(?:from|import)\s+["']([^"']+)["']/g), + (match) => match[1], + ); + if (!imports.includes("@earendil-works/pi-coding-agent")) { + failures.push("extension does not import the canonical @earendil-works/pi-coding-agent package"); + } + if (imports.some((specifier) => specifier.startsWith("@mariozechner/"))) { + failures.push("extension imports a legacy @mariozechner package"); + } + if (imports.some((specifier) => specifier.startsWith("@sinclair/typebox"))) { + failures.push('extension imports legacy "@sinclair/typebox" instead of "typebox"'); + } + } + if (output.extensionErrors.length > 0) failures.push("extension loader reported errors"); + if (!output.loadedExtensions.some(({ tools }) => tools.includes("hello"))) { + failures.push('no loaded extension registered the "hello" tool'); + } + if ( + !toolCalls.some( + (call) => + call.name === "hello" && + call.status === "ok" && + call.arguments?.name === "Bob" && + call.result === "Hello, Bob!", + ) + ) { + failures.push('no successful hello({ name: "Bob" }) call returned "Hello, Bob!"'); + } + if (output.response !== "Hello, Bob!") failures.push('final response was not exactly "Hello, Bob!"'); -const piExtensionsHarness = createPiCodingAgentHarness({ - name: "pi-coding-agent-extensions", - output: ({ response, session }) => { return { - response, - extensionPaths: session.extensionRunner.getExtensionPaths(), - toolDefinitions: session.getAllTools().map(({ name, description }) => ({ name, description })), + score: failures.length === 0 ? 1 : 0, + metadata: { + rationale: failures.length === 0 ? "Extension authoring workflow completed." : failures.join("; "), + }, }; }, +); + +const extensionHarnessTable = evalHarnessTable("Pi extension authoring system prompt", { + baseline: createExtensionAuthoringHarness("system-prompt-without-docs", excludeGuidelinesAndDocumentation), + candidate: createExtensionAuthoringHarness("default-system-prompt", prepareDefaultPromptOverride), }); -describeEval("Pi extensions", { harness: piExtensionsHarness }, (it) => { - it("creates, reloads, and uses a hello extension", async ({ run }) => { - const result = await run([ - { - type: "prompt", - content: - "Create a Pi extension with a hello tool that takes a name and returns a greeting. For example, passing Bob should return `Hello, Bob!`.", - }, - { type: "reload" }, - { - type: "prompt", - content: "Use the hello tool to greet Bob.", - }, - ]); - - expect(result.output.response).toBe("Hello, Bob!"); - expect(result.output.extensionPaths).toHaveLength(1); - expect(result.output.toolDefinitions).toContainEqual(expect.objectContaining({ name: "hello" })); - expect(result.errors).toEqual([]); - expect(toolCalls(result.session)).toContainEqual({ - name: "hello", - arguments: { name: "Bob" }, - status: "ok", - result: "Hello, Bob!", - }); - }); +describe.for(extensionHarnessTable)("$name", ({ harness }) => { + describeEval( + "Pi extension authoring system prompt", + { harness, judges: [ExtensionAuthoringJudge], judgeThreshold: null }, + (it) => { + it("creates, reloads, and uses a hello extension", async ({ run, task }) => { + const result = await run([ + { + type: "prompt", + content: + "Create a Pi extension with a hello tool that takes a name and returns a greeting. For example, passing Bob should return `Hello, Bob!`.", + }, + { type: "reload" }, + { + type: "prompt", + content: + "Use the hello tool to greet Bob. Respond with exactly the tool's greeting and nothing else.", + }, + ]); + if (result.output.extensionSource !== null) { + const runId = result.artifacts?.runId; + if (typeof runId !== "string") throw new Error("Pi eval run did not record a run ID."); + await recordEvalSourceArtifact(task, runId, { + name: "hello.ts", + contentType: "text/typescript", + body: result.output.extensionSource, + bodyEncoding: "utf-8", + }); + } + const expectsFullPrompt = harness.name === "default-system-prompt"; + expect(result.output.systemPromptHasGuidelines).toBe(expectsFullPrompt); + expect(result.output.systemPromptHasPiDocs).toBe(expectsFullPrompt); + }); + }, + ); }); diff --git a/packages/evals/src/pi-harness.ts b/packages/evals/src/pi-harness.ts index 8fbb0bb4a..a1cad3e16 100644 --- a/packages/evals/src/pi-harness.ts +++ b/packages/evals/src/pi-harness.ts @@ -1,7 +1,8 @@ -import assert from "node:assert"; -import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { performance } from "node:perf_hooks"; import { contentText } from "@earendil-works/pi-ai"; import { type AgentSession, @@ -15,29 +16,43 @@ import { import { createHarness, type Harness, + type HarnessContext, type JsonValue, normalizeRecord, type SimpleHarnessResult, type TranscriptEvent, toJsonValue, } from "vitest-evals/harness"; +import { PI_SESSION_SNAPSHOT_ARTIFACT } from "./vitest-evals/artifacts.ts"; -type PiCodingAgentInput = string | Array<{ type: "prompt"; content: string } | { type: "reload" }>; +export type PiCodingAgentInput = string | Array<{ type: "prompt"; content: string } | { type: "reload" }>; + +type PiCodingAgentModelSelection = { + provider: string; + id: string; +}; type PiCodingAgentHarnessOptions = { name?: string; + model?: PiCodingAgentModelSelection; noTools?: CreateAgentSessionOptions["noTools"]; + transformSystemPrompt?: (defaultPrompt: string) => string; }; type PiCodingAgentHarnessWithOutput = PiCodingAgentHarnessOptions & { output: (args: { response: string; session: AgentSession }) => TOutput | Promise; }; -function getRequiredModelSelection(): { provider: string; model: string } { - const provider = process.env.PI_PROVIDER?.trim(); - const model = process.env.PI_MODEL?.trim(); - if (!provider || !model) throw new Error("PI_PROVIDER and PI_MODEL must both be set for eval runs."); - return { provider, model }; +export function resolveModelSelection( + explicitModel: PiCodingAgentModelSelection | undefined, + environment: { PI_PROVIDER?: string; PI_MODEL?: string } = process.env, +): PiCodingAgentModelSelection { + const provider = (explicitModel?.provider ?? environment.PI_PROVIDER)?.trim(); + const id = (explicitModel?.id ?? environment.PI_MODEL)?.trim(); + if (!provider || !id) { + throw new Error("Select a harness model explicitly or set both PI_PROVIDER and PI_MODEL as defaults."); + } + return { provider, id }; } function toTranscriptEvents(messages: AgentSession["messages"]): TranscriptEvent[] { @@ -94,17 +109,21 @@ async function promptAgent(session: AgentSession, input: string, signal: AbortSi async function runPiCodingAgent( input: PiCodingAgentInput, signal: AbortSignal | undefined, + setArtifact: HarnessContext["setArtifact"], options: PiCodingAgentHarnessOptions | PiCodingAgentHarnessWithOutput, ): Promise> { + const startedAt = performance.now(); signal?.throwIfAborted(); - const selection = getRequiredModelSelection(); + const selection = resolveModelSelection(options.model); const modelRuntime = await ModelRuntime.create(); - const model = modelRuntime.getModel(selection.provider, selection.model); - if (!model) throw new Error(`Eval model not found: ${selection.provider}/${selection.model}`); + const model = modelRuntime.getModel(selection.provider, selection.id); + if (!model) throw new Error(`Eval model not found: ${selection.provider}/${selection.id}`); const root = await mkdtemp(join(tmpdir(), "pi-eval-")); const cwd = join(root, "workspace"); const agentDir = join(root, "agent"); + let transformedSystemPrompt: string | undefined; + let sessionManager: SessionManager | undefined; let session: AgentSession | undefined; let outcome: { success: true; result: SimpleHarnessResult } | { success: false; error: unknown }; try { @@ -114,12 +133,17 @@ async function runPiCodingAgent( agentDir, modelRuntime, settingsManager: SettingsManager.inMemory(), + ...(options.transformSystemPrompt + ? { resourceLoaderOptions: { systemPromptOverride: () => transformedSystemPrompt } } + : {}), }); signal?.throwIfAborted(); + sessionManager = SessionManager.create(cwd, join(root, "sessions")); + setArtifact("runId", sessionManager.getSessionId()); session = ( await createAgentSessionFromServices({ services, - sessionManager: SessionManager.inMemory(cwd), + sessionManager, model, thinkingLevel: "off", noTools: options.noTools, @@ -127,15 +151,21 @@ async function runPiCodingAgent( ).session; const evalSession = session; - const abort = () => void evalSession.abort(); + if (options.transformSystemPrompt) { + transformedSystemPrompt = options.transformSystemPrompt(evalSession.systemPrompt); + if (!transformedSystemPrompt.trim()) throw new Error("Transformed eval system prompt must not be empty."); + await evalSession.reload(); + } + let abortPromise: Promise | undefined; + const abort = () => { + abortPromise ??= evalSession.abort(); + }; signal?.addEventListener("abort", abort, { once: true }); try { signal?.throwIfAborted(); - assert.strictEqual( - evalSession.extensionRunner.getExtensionPaths().length, - 0, - "Expected an isolated eval session to start without extensions", - ); + if (evalSession.extensionRunner.getExtensionPaths().length !== 0) { + throw new Error("Expected an isolated eval session to start without extensions."); + } const steps = typeof input === "string" ? [{ type: "prompt" as const, content: input }] : input; let response: string | undefined; for (const step of steps) { @@ -148,6 +178,9 @@ async function runPiCodingAgent( if (response === undefined) throw new Error("Pi eval input must include at least one prompt step."); const output = "output" in options ? await options.output({ response, session: evalSession }) : response; const stats = evalSession.getSessionStats(); + const hasPricing = [model.cost, ...(model.cost.tiers ?? [])].some( + ({ input, output, cacheRead, cacheWrite }) => input > 0 || output > 0 || cacheRead > 0 || cacheWrite > 0, + ); outcome = { success: true, result: { @@ -160,17 +193,33 @@ async function runPiCodingAgent( outputTokens: stats.tokens.output, totalTokens: stats.tokens.total, toolCalls: stats.toolCalls, + metadata: { + cacheReadTokens: stats.tokens.cacheRead, + cacheWriteTokens: stats.tokens.cacheWrite, + ...(hasPricing ? { estimatedCostUsd: stats.cost } : {}), + }, }, }, }; } finally { signal?.removeEventListener("abort", abort); + if (abortPromise) await abortPromise; } } catch (error) { outcome = { success: false, error }; } const cleanupErrors: unknown[] = []; + if (sessionManager) { + try { + const sessionPath = sessionManager.getSessionFile(); + if (sessionPath && existsSync(sessionPath)) { + setArtifact(PI_SESSION_SNAPSHOT_ARTIFACT, await readFile(sessionPath, "utf8")); + } + } catch (error) { + cleanupErrors.push(error); + } + } try { session?.dispose(); } catch (error) { @@ -188,7 +237,10 @@ async function runPiCodingAgent( } if (cleanupErrors.length === 1) throw cleanupErrors[0]; if (cleanupErrors.length > 1) throw new AggregateError(cleanupErrors, "Agent cleanup failed."); - return outcome.result; + return { + ...outcome.result, + timings: { totalMs: performance.now() - startedAt }, + }; } export function createPiCodingAgentHarness( @@ -200,6 +252,6 @@ export function createPiCodingAgentHarness( ) { return createHarness({ name: options.name ?? "pi-coding-agent", - run: ({ input, signal }) => runPiCodingAgent(input, signal, options), + run: ({ input, signal, setArtifact }) => runPiCodingAgent(input, signal, setArtifact, options), }); } diff --git a/packages/evals/src/vitest-evals/artifacts.ts b/packages/evals/src/vitest-evals/artifacts.ts new file mode 100644 index 000000000..80780de5f --- /dev/null +++ b/packages/evals/src/vitest-evals/artifacts.ts @@ -0,0 +1,113 @@ +import { createHash } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import { basename, join, relative } from "node:path"; +import { + type RunnerTestCase, + recordArtifact, + type TestArtifact, + type TestArtifactBase, + type TestAttachment, +} from "vitest"; +import type { HarnessRun } from "vitest-evals/harness"; + +export const PI_SESSION_SNAPSHOT_ARTIFACT = "piSessionJsonl"; + +const evalSessionArtifactKey = Symbol("pi-evals-session-artifact"); +const evalSourceArtifactKey = Symbol("pi-evals-source-artifact"); + +interface PiSessionAttachment extends TestAttachment { + name: "session.jsonl"; + contentType: "application/jsonl"; + body: string; + bodyEncoding: "utf-8"; +} + +export interface SourceAttachment extends TestAttachment { + name: string; + contentType: string; + body: string; + bodyEncoding: "utf-8"; +} + +interface PiSessionArtifact extends TestArtifactBase { + type: "@earendil-works/pi-evals:session"; + runId: string; + attachments: [PiSessionAttachment] | []; +} + +interface SourceArtifact extends TestArtifactBase { + type: "@earendil-works/pi-evals:source"; + runId: string; + attachments: [SourceAttachment] | []; +} + +declare module "vitest" { + interface TestArtifactRegistry { + [evalSessionArtifactKey]: PiSessionArtifact; + [evalSourceArtifactKey]: SourceArtifact; + } +} + +export async function recordEvalSessionArtifact( + task: Readonly, + run: Pick, +): Promise { + const runId = run.artifacts?.runId; + const session = run.artifacts?.[PI_SESSION_SNAPSHOT_ARTIFACT]; + if (session === undefined) return; + if (typeof runId !== "string" || typeof session !== "string") { + throw new TypeError("Pi eval session artifact metadata is invalid."); + } + await recordArtifact(task, { + type: "@earendil-works/pi-evals:session", + runId, + attachments: [ + { + name: "session.jsonl", + contentType: "application/jsonl", + body: session, + bodyEncoding: "utf-8", + }, + ], + }); +} + +export async function recordEvalSourceArtifact( + task: Readonly, + runId: string, + attachment: SourceAttachment, +): Promise { + await recordArtifact(task, { + type: "@earendil-works/pi-evals:source", + runId, + attachments: [attachment], + }); +} + +export async function persistEvalArtifactReferences( + artifacts: ReadonlyArray, + runId: string, + artifactDirectory: string, +): Promise> { + const references: Array<{ name: string; path: string }> = []; + for (const artifact of artifacts) { + if ( + (artifact.type !== "@earendil-works/pi-evals:session" && + artifact.type !== "@earendil-works/pi-evals:source") || + artifact.runId !== runId + ) { + continue; + } + const category = artifact.type === "@earendil-works/pi-evals:session" ? "sessions" : "sources"; + for (const attachment of artifact.attachments) { + const name = basename(attachment.name); + if (name !== attachment.name) throw new TypeError(`Invalid eval artifact name: ${attachment.name}`); + const directory = join(artifactDirectory, category, createHash("sha256").update(runId).digest("hex")); + await mkdir(directory, { recursive: true, mode: 0o700 }); + const path = join(directory, name); + await writeFile(path, attachment.body, { encoding: "utf8", mode: 0o600 }); + references.push({ name, path: relative(artifactDirectory, path) }); + } + } + return references; +} diff --git a/packages/evals/src/vitest-evals/harness-table.ts b/packages/evals/src/vitest-evals/harness-table.ts new file mode 100644 index 000000000..4551cf9d8 --- /dev/null +++ b/packages/evals/src/vitest-evals/harness-table.ts @@ -0,0 +1,193 @@ +import { createHash } from "node:crypto"; +import { + attachHarnessRunToError, + getHarnessRunFromError, + type Harness, + type HarnessRun, + type JsonValue, +} from "vitest-evals/harness"; + +export const EVAL_HARNESS_ITERATION_ARTIFACT = "vitestEvalsHarnessIteration"; + +export type EvalHarnessIterationArtifact = { + schemaVersion: 1; + evalSet: string; + groupKey: string; + harness: string; + baseline: string; + candidates: string[]; + repetition: number; +}; + +export type EvalHarnessTableRow = { + harness: Harness; + name: string; + repetition: number; +}; + +export type EvalHarnessTablePairOptions = { + baseline: Harness; + candidate: Harness; + repetitions?: number; +}; + +export type EvalHarnessTableCandidatesOptions = { + baseline: Harness; + candidates: readonly Harness[]; + repetitions?: number; +}; + +export type EvalHarnessTableOptions = + | EvalHarnessTablePairOptions + | EvalHarnessTableCandidatesOptions; + +type EvalHarnessIterationPlan = Omit; + +export function parseEvalHarnessIterationArtifact( + value: JsonValue | undefined, +): EvalHarnessIterationArtifact | undefined { + if (value === null || value === undefined || typeof value !== "object" || Array.isArray(value)) return undefined; + const { schemaVersion, evalSet, groupKey, harness, baseline, candidates, repetition } = value; + if ( + schemaVersion !== 1 || + typeof evalSet !== "string" || + typeof groupKey !== "string" || + typeof harness !== "string" || + typeof baseline !== "string" || + !Array.isArray(candidates) || + !candidates.every((name): name is string => typeof name === "string") || + typeof repetition !== "number" + ) { + return undefined; + } + return { schemaVersion, evalSet, groupKey, harness, baseline, candidates, repetition }; +} + +function canonicalizeJson(value: unknown, ancestors: WeakSet): JsonValue { + if (value === null || typeof value === "string" || typeof value === "boolean") return value; + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new TypeError("Eval input must contain only finite numbers."); + return value; + } + if (typeof value !== "object") throw new TypeError("Eval input must be JSON-serializable."); + if (ancestors.has(value)) throw new TypeError("Eval input must not contain circular references."); + + ancestors.add(value); + try { + if (Array.isArray(value)) { + const result: JsonValue[] = []; + for (let index = 0; index < value.length; index += 1) { + if (!Object.hasOwn(value, index)) throw new TypeError("Eval input arrays must not be sparse."); + result.push(canonicalizeJson(value[index], ancestors)); + } + return result; + } + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + throw new TypeError("Eval input must contain only plain objects and arrays."); + } + const entries: Array<[string, unknown]> = Object.entries(value); + return Object.fromEntries( + entries + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) + .map(([key, item]): [string, JsonValue] => [key, canonicalizeJson(item, ancestors)]), + ); + } finally { + ancestors.delete(value); + } +} + +function deriveInputKey(input: unknown): string { + if (typeof input === "object" && input !== null && !Array.isArray(input) && "id" in input) { + const id = input.id; + if (typeof id === "string" && id.trim()) return id.trim(); + } + const canonicalInput = JSON.stringify(canonicalizeJson(input, new WeakSet())); + if (canonicalInput === undefined) throw new TypeError("Eval input must be JSON-serializable."); + return createHash("sha256").update(canonicalInput).digest("hex"); +} + +export function deriveEvalGroupKey(input: unknown, repetition: number): string { + return JSON.stringify([deriveInputKey(input), repetition]); +} + +function validateOptions( + evalSet: string, + baseline: Harness, + candidates: readonly Harness[], + repetitions: number, +): void { + if (!evalSet.trim()) throw new TypeError("evalSet must not be empty."); + if (candidates.length === 0) throw new TypeError("At least one candidate harness is required."); + const harnesses = [baseline, ...candidates]; + const names = new Set(harnesses.map((harness) => harness.name)); + if (names.size !== harnesses.length) throw new TypeError("Harness names must be unique within an eval set."); + if (!Number.isSafeInteger(repetitions) || repetitions < 1) { + throw new TypeError("repetitions must be a positive integer."); + } +} + +function withIterationArtifact( + harness: Harness, + plan: EvalHarnessIterationPlan, +): Harness { + return { + name: harness.name, + async run(input, context) { + const groupKey = deriveEvalGroupKey(input, plan.repetition); + const artifact: EvalHarnessIterationArtifact = { ...plan, groupKey }; + context.setArtifact(EVAL_HARNESS_ITERATION_ARTIFACT, artifact); + const attachIterationArtifact = (run: TRun): TRun => { + run.artifacts = { ...context.artifacts, ...run.artifacts, [EVAL_HARNESS_ITERATION_ARTIFACT]: artifact }; + return run; + }; + try { + return attachIterationArtifact(await harness.run(input, context)); + } catch (error) { + const partialRun = getHarnessRunFromError(error); + if (partialRun) { + throw attachHarnessRunToError(error, attachIterationArtifact(partialRun)); + } + throw error; + } + }, + }; +} + +export function evalHarnessTable( + evalSet: string, + options: EvalHarnessTablePairOptions, +): EvalHarnessTableRow[]; +export function evalHarnessTable( + evalSet: string, + options: EvalHarnessTableCandidatesOptions, +): EvalHarnessTableRow[]; +export function evalHarnessTable( + evalSet: string, + options: EvalHarnessTableOptions, +): EvalHarnessTableRow[] { + const repetitions = options.repetitions ?? 1; + const candidates = "candidate" in options ? [options.candidate] : options.candidates; + validateOptions(evalSet, options.baseline, candidates, repetitions); + + const rows: EvalHarnessTableRow[] = []; + const harnesses = [options.baseline, ...candidates]; + for (let repetition = 1; repetition <= repetitions; repetition += 1) { + for (const harness of harnesses) { + const plan: EvalHarnessIterationPlan = { + schemaVersion: 1, + evalSet, + harness: harness.name, + baseline: options.baseline.name, + candidates: candidates.map(({ name }) => name), + repetition, + }; + rows.push({ + harness: withIterationArtifact(harness, plan), + name: harness.name, + repetition, + }); + } + } + return rows; +} diff --git a/packages/evals/src/vitest-evals/reporter.ts b/packages/evals/src/vitest-evals/reporter.ts new file mode 100644 index 000000000..b1d279aa7 --- /dev/null +++ b/packages/evals/src/vitest-evals/reporter.ts @@ -0,0 +1,111 @@ +import { randomUUID } from "node:crypto"; +import { appendFile, mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import type { Reporter, SerializedError, TestCase, TestModule, TestRunEndReason, Vitest } from "vitest/node"; +import { isHarnessRun } from "vitest-evals/harness"; +import { PI_SESSION_SNAPSHOT_ARTIFACT, persistEvalArtifactReferences } from "./artifacts.ts"; +import { EVAL_HARNESS_ITERATION_ARTIFACT, parseEvalHarnessIterationArtifact } from "./harness-table.ts"; +import { formatHarnessComparisonReport, type HarnessObservation, summarizeHarnessComparisons } from "./summary.ts"; + +function readFiniteNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +} + +async function appendHarnessRunReport(test: TestCase): Promise { + const artifactDirectory = process.env.PI_EVAL_ARTIFACT_DIR?.trim(); + if (!artifactDirectory) return; + const harness = test.meta().harness; + if (!harness || !isHarnessRun(harness.run)) return; + + const run = harness.run; + const artifactRunId = run.artifacts?.runId; + const runId = typeof artifactRunId === "string" ? artifactRunId : randomUUID(); + const metadata = Object.fromEntries( + Object.entries(run.artifacts ?? {}).filter(([name]) => name !== "runId" && name !== PI_SESSION_SNAPSHOT_ARTIFACT), + ); + const record = { + schemaVersion: 1, + runId, + test: { + id: test.id, + file: test.module.relativeModuleId, + name: test.name, + fullName: test.fullName, + status: test.result().state, + }, + harness: harness.name, + usage: run.usage, + ...(run.timings ? { timings: run.timings } : {}), + ...(run.errors.length > 0 ? { errors: run.errors } : {}), + artifacts: await persistEvalArtifactReferences(test.artifacts(), runId, artifactDirectory), + ...(Object.keys(metadata).length > 0 ? { metadata } : {}), + }; + await mkdir(artifactDirectory, { recursive: true, mode: 0o700 }); + await appendFile(join(artifactDirectory, "runs.jsonl"), `${JSON.stringify(record)}\n`, { + encoding: "utf8", + flag: "a", + mode: 0o600, + }); +} + +function collectHarnessObservations(modules: ReadonlyArray): HarnessObservation[] { + const observations: HarnessObservation[] = []; + for (const module of modules) { + for (const test of module.children.allTests()) { + const harness = test.meta().harness; + if (!harness || !isHarnessRun(harness.run)) continue; + const run = harness.run; + const iteration = parseEvalHarnessIterationArtifact(run.artifacts?.[EVAL_HARNESS_ITERATION_ARTIFACT]); + if (!iteration) continue; + const score = readFiniteNumber(test.meta().eval?.avgScore); + const estimatedCostUsd = readFiniteNumber(run.usage.metadata?.estimatedCostUsd); + const observation = { + evalSet: iteration.evalSet, + groupKey: iteration.groupKey, + testName: test.name, + file: module.relativeModuleId, + harness: iteration.harness, + baseline: iteration.baseline, + candidates: iteration.candidates, + repetition: iteration.repetition, + ...(run.usage.totalTokens === undefined ? {} : { totalTokens: run.usage.totalTokens }), + ...(run.timings?.totalMs === undefined ? {} : { totalMs: run.timings.totalMs }), + ...(estimatedCostUsd === undefined ? {} : { estimatedCostUsd }), + }; + if (run.errors.length > 0) observations.push({ ...observation, outcome: "errored" }); + else if (score !== undefined) observations.push({ ...observation, outcome: "scored", score }); + else { + const state = test.result().state; + const outcome = state === "passed" ? "unscored" : state === "failed" ? "errored" : state; + observations.push({ ...observation, outcome }); + } + } + } + return observations; +} + +export default class EvalHarnessReporter implements Reporter { + private vitest: Vitest | undefined; + + onInit(vitest: Vitest): void { + this.vitest = vitest; + } + + async onTestCaseResult(test: TestCase): Promise { + await appendHarnessRunReport(test); + } + + onTestRunEnd( + modules: ReadonlyArray, + _errors: ReadonlyArray, + reason: TestRunEndReason, + ): void { + if (reason === "interrupted") { + this.vitest?.logger.log("\nEval comparisons unavailable: test run interrupted."); + return; + } + const report = summarizeHarnessComparisons(collectHarnessObservations(modules)); + const formatted = formatHarnessComparisonReport(report); + if (formatted) this.vitest?.logger.log(`\n${formatted}`); + } +} diff --git a/packages/evals/src/vitest-evals/setup.ts b/packages/evals/src/vitest-evals/setup.ts new file mode 100644 index 000000000..a711fea9b --- /dev/null +++ b/packages/evals/src/vitest-evals/setup.ts @@ -0,0 +1,8 @@ +import { afterEach } from "vitest"; +import type {} from "vitest-evals"; +import { recordEvalSessionArtifact } from "./artifacts.ts"; + +afterEach(async ({ task }) => { + const run = task.meta.harness?.run; + if (run) await recordEvalSessionArtifact(task, run); +}); diff --git a/packages/evals/src/vitest-evals/summary.ts b/packages/evals/src/vitest-evals/summary.ts new file mode 100644 index 000000000..1f2c4cb88 --- /dev/null +++ b/packages/evals/src/vitest-evals/summary.ts @@ -0,0 +1,438 @@ +import { styleText } from "node:util"; + +type HarnessObservationOutcome = "scored" | "unscored" | "skipped" | "pending" | "errored"; + +type HarnessObservationBase = { + evalSet: string; + groupKey: string; + testName: string; + file: string; + harness: string; + baseline: string; + candidates: string[]; + repetition: number; + totalTokens?: number; + totalMs?: number; + estimatedCostUsd?: number; +}; + +export type HarnessObservation = HarnessObservationBase & + ({ outcome: "scored"; score: number } | { outcome: Exclude; score?: never }); + +export type PairedMetricSummary = { + totalPairs: number; + eligiblePairs: number; + baselineMean: number | null; + candidateMean: number | null; + meanDelta: number | null; +}; + +export type CorrectnessLiftSummary = { + totalPairs: number; + eligiblePairs: number; + baselinePassRate: number | null; + candidatePassRate: number | null; + lift: number | null; + baselineWins: number; + candidateWins: number; + ties: number; +}; + +export type HarnessPairComparison = { + baseline: string; + candidate: string; + correctness: CorrectnessLiftSummary; + totalTokens: PairedMetricSummary; + totalMs: PairedMetricSummary; + estimatedCostUsd: PairedMetricSummary; +}; + +export type HarnessComparisonDiagnostic = { + evalSet: string; + groupKey: string; + testName: string; + file: string; + repetition: number; + harness: string; + reason: "missing-observation" | "duplicate-observation" | "harness-error" | "missing-score" | "unscorable-outcome"; +}; + +export type HarnessEvalSetReport = { + evalSet: string; + comparisons: HarnessPairComparison[]; +}; + +export type HarnessComparisonReport = { + schemaVersion: 1; + evalSets: HarnessEvalSetReport[]; + diagnostics: HarnessComparisonDiagnostic[]; +}; + +type HarnessDescriptor = { + name: string; + index: number; +}; + +type ObservationGroup = { + evalSet: string; + groupKey: string; + testName: string; + file: string; + repetition: number; + observationsByHarness: Map; +}; + +type EvalSetData = { + baseline: HarnessDescriptor; + candidatesByName: Map; + groupsByKey: Map; +}; + +type ObservationPair = { + baseline: HarnessObservation; + candidate: HarnessObservation; +}; + +function getOrCreate(map: Map, key: K, create: () => V): V { + const existing = map.get(key); + if (existing !== undefined) return existing; + const value = create(); + map.set(key, value); + return value; +} + +function mean(values: readonly number[]): number | null { + return values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length; +} + +function preciseDifference(left: number, right: number): number { + return Number((left - right).toPrecision(15)); +} + +function groupObservations(observations: readonly HarnessObservation[]): Map { + const evalSets = new Map(); + for (const observation of observations) { + const evalSet = getOrCreate(evalSets, observation.evalSet, () => ({ + baseline: { name: observation.baseline, index: 0 }, + candidatesByName: new Map(), + groupsByKey: new Map(), + })); + + for (const [index, name] of observation.candidates.entries()) { + const existing = evalSet.candidatesByName.get(name); + if (!existing || index < existing.index) evalSet.candidatesByName.set(name, { name, index }); + } + + const group = getOrCreate( + evalSet.groupsByKey, + JSON.stringify([observation.file, observation.testName, observation.groupKey]), + () => ({ + evalSet: observation.evalSet, + groupKey: observation.groupKey, + testName: observation.testName, + file: observation.file, + repetition: observation.repetition, + observationsByHarness: new Map(), + }), + ); + getOrCreate(group.observationsByHarness, observation.harness, (): HarnessObservation[] => []).push(observation); + } + return evalSets; +} + +function orderedHarnesses(evalSet: EvalSetData): HarnessDescriptor[] { + return [ + evalSet.baseline, + ...[...evalSet.candidatesByName.values()].sort( + (left, right) => left.index - right.index || left.name.localeCompare(right.name), + ), + ]; +} + +function orderedCandidates(evalSet: EvalSetData): HarnessDescriptor[] { + return [...evalSet.candidatesByName.values()].sort( + (left, right) => left.index - right.index || left.name.localeCompare(right.name), + ); +} + +function orderedGroups(evalSet: EvalSetData): ObservationGroup[] { + return [...evalSet.groupsByKey.values()].sort( + (left, right) => left.groupKey.localeCompare(right.groupKey) || left.repetition - right.repetition, + ); +} + +function collectDiagnostics( + harnesses: readonly HarnessDescriptor[], + groups: readonly ObservationGroup[], +): HarnessComparisonDiagnostic[] { + const diagnostics: HarnessComparisonDiagnostic[] = []; + for (const group of groups) { + for (const { name: harness } of harnesses) { + const observations = group.observationsByHarness.get(harness) ?? []; + let reason: HarnessComparisonDiagnostic["reason"] | undefined; + if (observations.length === 0) reason = "missing-observation"; + else if (observations.length > 1) reason = "duplicate-observation"; + else if (observations[0].outcome === "errored") reason = "harness-error"; + else if (observations[0].outcome === "unscored") { + reason = "missing-score"; + } else if (observations[0].outcome !== "scored") { + reason = "unscorable-outcome"; + } + if (!reason) continue; + diagnostics.push({ + evalSet: group.evalSet, + groupKey: group.groupKey, + testName: group.testName, + file: group.file, + repetition: group.repetition, + harness, + reason, + }); + } + } + return diagnostics; +} + +function pairObservations( + groups: readonly ObservationGroup[], + baselineHarness: string, + candidateHarness: string, +): ObservationPair[] { + const pairs: ObservationPair[] = []; + for (const group of groups) { + const baseline = group.observationsByHarness.get(baselineHarness) ?? []; + const candidate = group.observationsByHarness.get(candidateHarness) ?? []; + if (baseline.length === 1 && candidate.length === 1) { + pairs.push({ baseline: baseline[0], candidate: candidate[0] }); + } + } + return pairs; +} + +function summarizeMetric( + pairs: readonly ObservationPair[], + select: (observation: HarnessObservation) => number | undefined, + totalPairs: number, +): PairedMetricSummary { + const baselineValues: number[] = []; + const candidateValues: number[] = []; + for (const { baseline, candidate } of pairs) { + if (baseline.outcome !== "scored" || candidate.outcome !== "scored") continue; + const baselineValue = select(baseline); + const candidateValue = select(candidate); + if ( + baselineValue === undefined || + candidateValue === undefined || + !Number.isFinite(baselineValue) || + !Number.isFinite(candidateValue) + ) { + continue; + } + baselineValues.push(baselineValue); + candidateValues.push(candidateValue); + } + + const baselineMean = mean(baselineValues); + const candidateMean = mean(candidateValues); + return { + totalPairs, + eligiblePairs: baselineValues.length, + baselineMean, + candidateMean, + meanDelta: + baselineMean === null || candidateMean === null ? null : preciseDifference(candidateMean, baselineMean), + }; +} + +function summarizeCorrectness(pairs: readonly ObservationPair[], totalPairs: number): CorrectnessLiftSummary { + let eligiblePairs = 0; + let baselinePasses = 0; + let candidatePasses = 0; + let baselineWins = 0; + let candidateWins = 0; + let ties = 0; + + for (const { baseline, candidate } of pairs) { + if (baseline.outcome !== "scored" || candidate.outcome !== "scored") continue; + eligiblePairs += 1; + const baselinePassed = baseline.score >= 1; + const candidatePassed = candidate.score >= 1; + if (baselinePassed) baselinePasses += 1; + if (candidatePassed) candidatePasses += 1; + if (baselinePassed === candidatePassed) ties += 1; + else if (baselinePassed) baselineWins += 1; + else candidateWins += 1; + } + + const baselinePassRate = eligiblePairs === 0 ? null : baselinePasses / eligiblePairs; + const candidatePassRate = eligiblePairs === 0 ? null : candidatePasses / eligiblePairs; + return { + totalPairs, + eligiblePairs, + baselinePassRate, + candidatePassRate, + lift: + baselinePassRate === null || candidatePassRate === null + ? null + : preciseDifference(candidatePassRate, baselinePassRate), + baselineWins, + candidateWins, + ties, + }; +} + +function compareHarnesses( + baseline: HarnessDescriptor, + candidate: HarnessDescriptor, + groups: readonly ObservationGroup[], +): HarnessPairComparison { + const pairs = pairObservations(groups, baseline.name, candidate.name); + return { + baseline: baseline.name, + candidate: candidate.name, + correctness: summarizeCorrectness(pairs, groups.length), + totalTokens: summarizeMetric(pairs, ({ totalTokens }) => totalTokens, groups.length), + totalMs: summarizeMetric(pairs, ({ totalMs }) => totalMs, groups.length), + estimatedCostUsd: summarizeMetric(pairs, ({ estimatedCostUsd }) => estimatedCostUsd, groups.length), + }; +} + +export function summarizeHarnessComparisons(observations: readonly HarnessObservation[]): HarnessComparisonReport { + const evalSets: HarnessEvalSetReport[] = []; + const diagnostics: HarnessComparisonDiagnostic[] = []; + for (const [evalSet, data] of [...groupObservations(observations)].sort(([left], [right]) => + left.localeCompare(right), + )) { + const harnesses = orderedHarnesses(data); + const candidates = orderedCandidates(data); + const groups = orderedGroups(data); + evalSets.push({ + evalSet, + comparisons: candidates.map((candidate) => compareHarnesses(data.baseline, candidate, groups)), + }); + diagnostics.push(...collectDiagnostics(harnesses, groups)); + } + + return { + schemaVersion: 1, + evalSets, + diagnostics: diagnostics.sort( + (left, right) => + left.evalSet.localeCompare(right.evalSet) || + left.file.localeCompare(right.file) || + left.groupKey.localeCompare(right.groupKey) || + left.repetition - right.repetition || + left.harness.localeCompare(right.harness), + ), + }; +} + +function formatPercentage(value: number | null): string { + return value === null ? "unavailable" : `${(value * 100).toFixed(1)}%`; +} + +function formatSigned(value: number, fractionDigits: number): string { + return `${value >= 0 ? "+" : ""}${value.toFixed(fractionDigits)}`; +} + +function formatCoverage(eligiblePairs: number, totalPairs: number): string { + return styleText("gray", `(${eligiblePairs}/${totalPairs} pairs)`); +} + +function formatReportLine(label: string, value: string): string { + return ` ${styleText("gray", label.padStart(9))} ${value}`; +} + +function colorDelta(value: number, formatted: string, positiveIsBetter: boolean): string { + if (value === 0) return styleText("gray", formatted); + const improved = positiveIsBetter ? value > 0 : value < 0; + return styleText(improved ? "green" : "red", formatted); +} + +function formatMetric( + label: string, + metric: PairedMetricSummary, + formatValue: (value: number) => string, + formatDelta: (value: number) => string, + comparisonPairs: number, +): string { + const coverage = + metric.eligiblePairs === 0 || metric.eligiblePairs === comparisonPairs + ? "" + : ` ${formatCoverage(metric.eligiblePairs, metric.totalPairs)}`; + if (metric.baselineMean === null || metric.candidateMean === null || metric.meanDelta === null) { + return formatReportLine(label, `${styleText("yellow", "unavailable")}${coverage}`); + } + const delta = colorDelta(metric.meanDelta, formatDelta(metric.meanDelta), false); + const values = styleText( + "gray", + `(candidate ${formatValue(metric.candidateMean)}, baseline ${formatValue(metric.baselineMean)})`, + ); + return formatReportLine(label, `${delta} ${values}${coverage}`); +} + +export function formatHarnessComparisonReport(report: HarnessComparisonReport): string { + if (report.evalSets.every(({ comparisons }) => comparisons.length === 0)) return ""; + const lines = [styleText("bold", "Eval Comparisons")]; + for (const evalSet of report.evalSets) { + lines.push(` ${evalSet.evalSet}`); + for (const [index, comparison] of evalSet.comparisons.entries()) { + if (index > 0) lines.push(""); + const { correctness } = comparison; + lines.push(formatReportLine("Baseline", comparison.baseline)); + lines.push( + formatReportLine( + "Candidate", + `${comparison.candidate} ${formatCoverage(correctness.eligiblePairs, correctness.totalPairs)}`, + ), + ); + if (correctness.lift === null) { + lines.push(formatReportLine("Pass rate", styleText("yellow", "unavailable"))); + } else { + const lift = correctness.lift * 100; + const delta = colorDelta(lift, `${formatSigned(lift, 1)} pp`, true); + const values = styleText( + "gray", + `(candidate ${formatPercentage(correctness.candidatePassRate)}, baseline ${formatPercentage(correctness.baselinePassRate)})`, + ); + lines.push(formatReportLine("Pass rate", `${delta} ${values}`)); + } + lines.push( + formatMetric( + "Tokens", + comparison.totalTokens, + (value) => value.toFixed(1), + (value) => formatSigned(value, 1), + correctness.eligiblePairs, + ), + ); + lines.push( + formatMetric( + "Latency", + comparison.totalMs, + (value) => `${value.toFixed(1)}ms`, + (value) => `${formatSigned(value, 1)}ms`, + correctness.eligiblePairs, + ), + ); + lines.push( + formatMetric( + "Est. cost", + comparison.estimatedCostUsd, + (value) => `$${value.toFixed(4)}`, + (value) => `${value >= 0 ? "+" : "-"}$${Math.abs(value).toFixed(4)}`, + correctness.eligiblePairs, + ), + ); + } + } + if (report.diagnostics.length > 0) { + lines.push(` ${styleText("yellow", "Incomplete observations")}`); + for (const diagnostic of report.diagnostics) { + lines.push( + ` ${diagnostic.reason}: ${diagnostic.file}/${diagnostic.testName} repetition ${diagnostic.repetition}, harness ${diagnostic.harness}`, + ); + } + } + return lines.join("\n"); +} diff --git a/packages/evals/test/pi-harness.test.ts b/packages/evals/test/pi-harness.test.ts new file mode 100644 index 000000000..13fb42426 --- /dev/null +++ b/packages/evals/test/pi-harness.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { resolveModelSelection } from "../src/pi-harness.ts"; + +describe("resolveModelSelection", () => { + it("prefers an explicit harness model over environment defaults", () => { + expect( + resolveModelSelection( + { provider: "anthropic", id: "claude-opus-4-6" }, + { PI_PROVIDER: "openai-codex", PI_MODEL: "gpt-5.6-sol" }, + ), + ).toEqual({ provider: "anthropic", id: "claude-opus-4-6" }); + }); + + it("uses trimmed environment defaults when the harness has no explicit model", () => { + expect(resolveModelSelection(undefined, { PI_PROVIDER: " openai-codex ", PI_MODEL: " gpt-5.6-sol " })).toEqual({ + provider: "openai-codex", + id: "gpt-5.6-sol", + }); + }); + + it.each([ + [undefined, {}], + [undefined, { PI_PROVIDER: "openai-codex" }], + [undefined, { PI_MODEL: "gpt-5.6-sol" }], + [ + { provider: "", id: "gpt-5.6-sol" }, + { PI_PROVIDER: "openai-codex", PI_MODEL: "gpt-5.6-sol" }, + ], + ] as const)("rejects an incomplete model selection", (explicitModel, environment) => { + expect(() => resolveModelSelection(explicitModel, environment)).toThrow( + "Select a harness model explicitly or set both PI_PROVIDER and PI_MODEL as defaults.", + ); + }); +}); diff --git a/packages/evals/test/vitest-evals/artifacts.test.ts b/packages/evals/test/vitest-evals/artifacts.test.ts new file mode 100644 index 000000000..67e1f6275 --- /dev/null +++ b/packages/evals/test/vitest-evals/artifacts.test.ts @@ -0,0 +1,103 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { + persistEvalArtifactReferences, + recordEvalSessionArtifact, + recordEvalSourceArtifact, +} from "../../src/vitest-evals/artifacts.ts"; + +it("records session and source artifacts against the explicit test task", async ({ task }) => { + const runId = "run-1"; + await recordEvalSessionArtifact(task, { + artifacts: { runId, piSessionJsonl: '{"type":"session"}\n' }, + }); + await recordEvalSourceArtifact(task, runId, { + name: "hello.ts", + contentType: "text/typescript", + body: "export default function () {}\n", + bodyEncoding: "utf-8", + }); + + expect(task.artifacts).toContainEqual( + expect.objectContaining({ + type: "@earendil-works/pi-evals:session", + runId, + attachments: [ + expect.objectContaining({ + name: "session.jsonl", + body: '{"type":"session"}\n', + bodyEncoding: "utf-8", + contentType: "application/jsonl", + }), + ], + }), + ); + expect(task.artifacts).toContainEqual( + expect.objectContaining({ + type: "@earendil-works/pi-evals:source", + runId, + attachments: [ + expect.objectContaining({ + name: "hello.ts", + body: "export default function () {}\n", + bodyEncoding: "utf-8", + contentType: "text/typescript", + }), + ], + }), + ); +}); + +it("persists and selects attachments belonging to the reported run", async () => { + const root = await mkdtemp(join(tmpdir(), "pi-eval-artifact-report-test-")); + try { + const references = await persistEvalArtifactReferences( + [ + { + type: "@earendil-works/pi-evals:session", + runId: "run-1", + attachments: [ + { + name: "session.jsonl", + body: '{"type":"session"}\n', + bodyEncoding: "utf-8", + contentType: "application/jsonl", + }, + ], + }, + { + type: "@earendil-works/pi-evals:session", + runId: "run-2", + attachments: [], + }, + { + type: "@earendil-works/pi-evals:source", + runId: "run-1", + attachments: [ + { + name: "hello.ts", + body: "export default function () {}\n", + bodyEncoding: "utf-8", + contentType: "text/typescript", + }, + ], + }, + { type: "internal:annotation", annotation: { message: "other", type: "info" } }, + ], + "run-1", + root, + ); + expect(references).toEqual([ + { name: "session.jsonl", path: expect.stringMatching(/^sessions\/[a-f0-9]{64}\/session\.jsonl$/) }, + { name: "hello.ts", path: expect.stringMatching(/^sources\/[a-f0-9]{64}\/hello\.ts$/) }, + ]); + for (const { name, path } of references) { + const expected = name === "session.jsonl" ? '{"type":"session"}\n' : "export default function () {}\n"; + expect(await readFile(join(root, path), "utf8")).toBe(expected); + } + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/packages/evals/test/vitest-evals/harness-table.test.ts b/packages/evals/test/vitest-evals/harness-table.test.ts new file mode 100644 index 000000000..13ad6ab8d --- /dev/null +++ b/packages/evals/test/vitest-evals/harness-table.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from "vitest"; +import { createHarness, type HarnessContext } from "vitest-evals/harness"; +import { + deriveEvalGroupKey, + EVAL_HARNESS_ITERATION_ARTIFACT, + evalHarnessTable, + parseEvalHarnessIterationArtifact, +} from "../../src/vitest-evals/harness-table.ts"; + +describe("deriveEvalGroupKey", () => { + it("combines a trimmed string input ID with repetition", () => { + expect(deriveEvalGroupKey({ id: " input-1 ", prompt: "hello" }, 2)).toBe(JSON.stringify(["input-1", 2])); + }); + + it("hashes canonical JSON independently of object key order", () => { + expect(deriveEvalGroupKey({ first: 1, second: [true, "value"] }, 1)).toBe( + deriveEvalGroupKey({ second: [true, "value"], first: 1 }, 1), + ); + expect(deriveEvalGroupKey({ first: 1 }, 1)).not.toBe(deriveEvalGroupKey({ first: 2 }, 1)); + expect(deriveEvalGroupKey({ first: 1 }, 1)).not.toBe(deriveEvalGroupKey({ first: 1 }, 2)); + expect(deriveEvalGroupKey(["first", "second"], 1)).not.toBe(deriveEvalGroupKey(["second", "first"], 1)); + }); + + it("rejects non-JSON and circular input", () => { + const circular: { self?: unknown } = {}; + circular.self = circular; + expect(() => deriveEvalGroupKey(new Date(0), 1)).toThrow("only plain objects and arrays"); + expect(() => deriveEvalGroupKey(Array(1), 1)).toThrow("must not be sparse"); + expect(() => deriveEvalGroupKey(circular, 1)).toThrow("must not contain circular references"); + }); +}); + +function createFakeHarness(name: string) { + return createHarness<{ id: string }, { harness: string; inputId: string }>({ + name, + run: ({ input }) => ({ + output: { harness: name, inputId: input.id }, + events: [ + { type: "message", role: "user", content: input.id }, + { type: "message", role: "assistant", content: name }, + ], + }), + }); +} + +const harnessTable = evalHarnessTable("local multi-harness eval", { + baseline: createFakeHarness("withoutSkill"), + candidates: [createFakeHarness("withSkill")], + repetitions: 2, +}); + +describe("evalHarnessTable", () => { + it("plans repetitions in declaration order", () => { + expect(harnessTable.map(({ name, repetition }) => ({ name, repetition }))).toEqual([ + { name: "withoutSkill", repetition: 1 }, + { name: "withSkill", repetition: 1 }, + { name: "withoutSkill", repetition: 2 }, + { name: "withSkill", repetition: 2 }, + ]); + }); + + it("accepts a singular candidate", () => { + const rows = evalHarnessTable("singular candidate", { + baseline: createFakeHarness("baseline"), + candidate: createFakeHarness("candidate"), + }); + + expect(rows.map(({ name }) => name)).toEqual(["baseline", "candidate"]); + }); + + it("attaches iteration metadata to every wrapped harness run", async () => { + for (const row of harnessTable) { + const artifacts: HarnessContext["artifacts"] = {}; + const context: HarnessContext = { + artifacts, + setArtifact(name, value) { + artifacts[name] = value; + }, + }; + const result = await row.harness.run({ id: "first" }, context); + + expect(result.output).toEqual({ harness: row.name, inputId: "first" }); + expect(parseEvalHarnessIterationArtifact(result.artifacts?.[EVAL_HARNESS_ITERATION_ARTIFACT])).toEqual({ + schemaVersion: 1, + evalSet: "local multi-harness eval", + groupKey: deriveEvalGroupKey({ id: "first" }, row.repetition), + harness: row.name, + baseline: "withoutSkill", + candidates: ["withSkill"], + repetition: row.repetition, + }); + } + }); +}); diff --git a/packages/evals/test/vitest-evals/summary.test.ts b/packages/evals/test/vitest-evals/summary.test.ts new file mode 100644 index 000000000..05b2f7a89 --- /dev/null +++ b/packages/evals/test/vitest-evals/summary.test.ts @@ -0,0 +1,232 @@ +import { stripVTControlCharacters } from "node:util"; +import { describe, expect, it } from "vitest"; +import { + formatHarnessComparisonReport, + type HarnessObservation, + summarizeHarnessComparisons, +} from "../../src/vitest-evals/summary.ts"; + +type ObservationResult = "passed" | "failed" | Exclude; + +function observation( + harness: string, + testName: string, + result: ObservationResult, + metrics: Pick = {}, + baseline = "without-tools", + candidates: string[] = ["with-tools"], +): HarnessObservation { + const base = { + evalSet: "tool access", + groupKey: JSON.stringify([testName, 1]), + testName, + file: "src/tool-access.eval.ts", + harness, + baseline, + candidates, + repetition: 1, + ...metrics, + }; + if (result === "passed" || result === "failed") { + return { ...base, outcome: "scored", score: result === "passed" ? 1 : 0 }; + } + return { ...base, outcome: result }; +} + +describe("summarizeHarnessComparisons", () => { + it("computes paired correctness lift separately from efficiency deltas", () => { + const report = summarizeHarnessComparisons([ + observation("without-tools", "create", "failed", { + totalTokens: 100, + totalMs: 1000, + estimatedCostUsd: 0.01, + }), + observation("with-tools", "create", "passed", { + totalTokens: 120, + totalMs: 800, + estimatedCostUsd: 0.02, + }), + observation("without-tools", "inspect", "passed", { totalTokens: 200 }), + observation("with-tools", "inspect", "passed", { totalTokens: 180 }), + ]); + + expect(report.evalSets).toHaveLength(1); + expect(report.evalSets[0]?.comparisons).toEqual([ + expect.objectContaining({ + baseline: "without-tools", + candidate: "with-tools", + correctness: { + totalPairs: 2, + eligiblePairs: 2, + baselinePassRate: 0.5, + candidatePassRate: 1, + lift: 0.5, + baselineWins: 0, + candidateWins: 1, + ties: 1, + }, + totalTokens: { + totalPairs: 2, + eligiblePairs: 2, + baselineMean: 150, + candidateMean: 150, + meanDelta: 0, + }, + totalMs: { + totalPairs: 2, + eligiblePairs: 1, + baselineMean: 1000, + candidateMean: 800, + meanDelta: -200, + }, + estimatedCostUsd: { + totalPairs: 2, + eligiblePairs: 1, + baselineMean: 0.01, + candidateMean: 0.02, + meanDelta: 0.01, + }, + }), + ]); + expect(report.diagnostics).toEqual([]); + }); + + it("reports missing observations without coercing them to failures or zero telemetry", () => { + const report = summarizeHarnessComparisons([ + observation("without-tools", "create", "failed"), + observation("with-tools", "create", "passed"), + observation("without-tools", "inspect", "passed"), + ]); + const comparison = report.evalSets[0]?.comparisons[0]; + + expect(comparison?.correctness).toEqual({ + totalPairs: 2, + eligiblePairs: 1, + baselinePassRate: 0, + candidatePassRate: 1, + lift: 1, + baselineWins: 0, + candidateWins: 1, + ties: 0, + }); + expect(comparison?.totalTokens).toEqual({ + totalPairs: 2, + eligiblePairs: 0, + baselineMean: null, + candidateMean: null, + meanDelta: null, + }); + expect(report.diagnostics).toContainEqual( + expect.objectContaining({ + testName: "inspect", + harness: "with-tools", + reason: "missing-observation", + }), + ); + }); + + it("keeps identical inputs in different test files separate", () => { + const report = summarizeHarnessComparisons([ + observation("without-tools", "shared", "failed"), + observation("with-tools", "shared", "passed"), + { ...observation("without-tools", "shared", "passed"), file: "src/other.eval.ts" }, + { ...observation("with-tools", "shared", "passed"), file: "src/other.eval.ts" }, + ]); + + expect(report.evalSets[0]?.comparisons[0]?.correctness).toEqual( + expect.objectContaining({ totalPairs: 2, eligiblePairs: 2 }), + ); + expect(report.diagnostics).toEqual([]); + }); + + it("does not score harness errors as correctness failures", () => { + const report = summarizeHarnessComparisons([ + observation("without-tools", "create", "errored", { totalTokens: 100 }), + observation("with-tools", "create", "passed", { totalTokens: 100 }), + ]); + + expect(report.evalSets[0]?.comparisons[0]?.correctness).toEqual( + expect.objectContaining({ totalPairs: 1, eligiblePairs: 0 }), + ); + expect(report.evalSets[0]?.comparisons[0]?.totalTokens.eligiblePairs).toBe(0); + expect(report.diagnostics).toContainEqual( + expect.objectContaining({ harness: "without-tools", reason: "harness-error" }), + ); + }); + + it("does not derive correctness from completed Vitest tests without judge scores", () => { + const withoutScore = observation("without-tools", "create", "unscored"); + const withScore = observation("with-tools", "create", "unscored"); + + const report = summarizeHarnessComparisons([withoutScore, withScore]); + + expect(report.evalSets[0]?.comparisons[0]?.correctness.eligiblePairs).toBe(0); + expect(report.diagnostics).toEqual([ + expect.objectContaining({ harness: "with-tools", reason: "missing-score" }), + expect.objectContaining({ harness: "without-tools", reason: "missing-score" }), + ]); + }); + + it("compares each candidate with the declared baseline", () => { + const candidates = ["second", "third"]; + const report = summarizeHarnessComparisons([ + observation("first", "input", "passed", {}, "first", candidates), + observation("second", "input", "passed", {}, "first", candidates), + observation("third", "input", "passed", {}, "first", candidates), + ]); + + expect(report.evalSets[0]?.comparisons.map(({ baseline, candidate }) => [baseline, candidate])).toEqual([ + ["first", "second"], + ["first", "third"], + ]); + }); + + it("retains a declared harness with no completed observations", () => { + const report = summarizeHarnessComparisons([observation("without-tools", "create", "failed")]); + + expect(report.evalSets[0]?.comparisons).toHaveLength(1); + expect(report.evalSets[0]?.comparisons[0]?.correctness.eligiblePairs).toBe(0); + expect(report.diagnostics).toContainEqual( + expect.objectContaining({ + testName: "create", + harness: "with-tools", + reason: "missing-observation", + }), + ); + }); + + it("reports duplicate and unscorable observations once across multiple harness pairs", () => { + const candidates = ["second", "third"]; + const report = summarizeHarnessComparisons([ + observation("first", "duplicate", "passed", {}, "first", candidates), + observation("first", "duplicate", "failed", {}, "first", candidates), + observation("second", "duplicate", "passed", {}, "first", candidates), + observation("third", "duplicate", "passed", {}, "first", candidates), + observation("first", "skipped", "skipped", {}, "first", candidates), + observation("second", "skipped", "passed", {}, "first", candidates), + observation("third", "skipped", "passed", {}, "first", candidates), + ]); + + expect(report.diagnostics.filter(({ reason }) => reason === "duplicate-observation")).toEqual([ + expect.objectContaining({ testName: "duplicate", harness: "first" }), + ]); + expect(report.diagnostics.filter(({ reason }) => reason === "unscorable-outcome")).toEqual([ + expect.objectContaining({ testName: "skipped", harness: "first" }), + ]); + }); + + it("formats lift and telemetry availability for the terminal report", () => { + const report = summarizeHarnessComparisons([ + observation("without-tools", "create", "failed", { totalMs: 34853.7 }), + observation("with-tools", "create", "passed", { totalMs: 30694.2 }), + ]); + + const formatted = stripVTControlCharacters(formatHarnessComparisonReport(report)); + expect(formatted).toContain("Eval Comparisons"); + expect(formatted).toContain(" Baseline without-tools"); + expect(formatted).toContain("Candidate with-tools (1/1 pairs)"); + expect(formatted).toContain("Pass rate +100.0 pp (candidate 100.0%, baseline 0.0%)"); + expect(formatted).toContain(" Tokens unavailable"); + expect(formatted).toContain(" Latency -4159.5ms (candidate 30694.2ms, baseline 34853.7ms)"); + }); +}); diff --git a/packages/evals/vitest.config.ts b/packages/evals/vitest.config.ts index 5dd4fc817..67dad900b 100644 --- a/packages/evals/vitest.config.ts +++ b/packages/evals/vitest.config.ts @@ -10,7 +10,8 @@ export default mergeConfig( include: ["src/**/*.eval.ts"], testTimeout: 120000, hookTimeout: 30000, - reporters: ["vitest-evals/reporter"], + setupFiles: ["./src/vitest-evals/setup.ts"], + reporters: ["vitest-evals/reporter", "./src/vitest-evals/reporter.ts"], }, resolve: { alias: [{ find: /^@earendil-works\/pi-coding-agent$/, replacement: workspaceSourcePaths.codingAgentIndex }], diff --git a/packages/evals/vitest.test.config.ts b/packages/evals/vitest.test.config.ts new file mode 100644 index 000000000..20aa79309 --- /dev/null +++ b/packages/evals/vitest.test.config.ts @@ -0,0 +1,14 @@ +import { defineConfig, mergeConfig } from "vitest/config"; +import baseConfig, { workspaceSourcePaths } from "../../vitest.base.ts"; + +export default mergeConfig( + baseConfig, + defineConfig({ + test: { + include: ["test/**/*.test.ts"], + }, + resolve: { + alias: [{ find: /^@earendil-works\/pi-coding-agent$/, replacement: workspaceSourcePaths.codingAgentIndex }], + }, + }), +);