mirror of
https://github.com/badlogic/pi-mono.git
synced 2026-11-05 23:37:25 +00:00
feat(coding-agent): add comparative Pi eval harness
This commit is contained in:
Christian Klotz
parent
d7b02636a0
commit
32f3a97282
17 changed files with 1708 additions and 90 deletions
|
|
@ -73,8 +73,10 @@ const VIRTUAL_MODULES: Record<string, unknown> = {
|
|||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
const isTypeScriptSourceRuntime = !isBunBinary && path.extname(fileURLToPath(import.meta.url)) === ".ts";
|
||||
|
||||
/**
|
||||
* Get aliases for jiti (used in Node.js/development mode).
|
||||
* Get aliases for jiti (used in built Node.js mode).
|
||||
* In Bun binary mode, virtualModules is used instead.
|
||||
*/
|
||||
let _aliases: Record<string, string> | null = null;
|
||||
|
|
@ -410,10 +412,13 @@ async function loadExtensionModule(extensionPath: string, cacheToken?: Extension
|
|||
|
||||
const jiti = createJiti(import.meta.url, {
|
||||
moduleCache: false,
|
||||
// In Bun binary: use virtualModules for bundled packages (no filesystem resolution)
|
||||
// Also disable tryNative so jiti handles ALL imports (not just the entry point)
|
||||
// In Node.js/dev: use aliases to resolve to node_modules paths
|
||||
...(isBunBinary ? { virtualModules: VIRTUAL_MODULES, tryNative: false } : { alias: getAliases() }),
|
||||
// Bun uses modules embedded in the executable. Source TypeScript reuses the
|
||||
// host-resolved modules and root tsconfig paths. Built Node uses dist aliases.
|
||||
...(isBunBinary
|
||||
? { virtualModules: VIRTUAL_MODULES, tryNative: false }
|
||||
: isTypeScriptSourceRuntime
|
||||
? { virtualModules: VIRTUAL_MODULES, tsconfigPaths: true }
|
||||
: { alias: getAliases() }),
|
||||
});
|
||||
|
||||
const module = await jiti.import(extensionPath, { default: true });
|
||||
|
|
|
|||
1
packages/evals/.gitignore
vendored
Normal file
1
packages/evals/.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
.eval/
|
||||
|
|
@ -1,32 +1,150 @@
|
|||
# Pi evals
|
||||
|
||||
Behavioral evaluations for Pi using `vitest-evals`.
|
||||
Pi evals are behavioral, model-backed checks for Pi workflows. They adapt a real `AgentSession` to `vitest-evals`, run
|
||||
it in isolated temporary project and agent directories, and attach native Pi session artifacts.
|
||||
Use them to measure end-to-end behavior and compare prompts, tools, skills, models, or other harness configurations.
|
||||
|
||||
`src/pi-harness.ts` adapts Pi's `AgentSession` directly to `vitest-evals`. Each run uses isolated temporary project and
|
||||
agent directories that are removed afterward. Judge harnesses are configured separately from the application harness.
|
||||
## Running evals
|
||||
|
||||
Eval cases supply either one prompt or a JSON-safe sequence of prompt and reload steps. The harness executes that input
|
||||
and normalizes the resulting session without defining scenario-specific behavior. Suites can configure a typed output
|
||||
selector over the final response and `AgentSession` state.
|
||||
|
||||
## Running
|
||||
|
||||
From the repository root, run with an explicit provider and model:
|
||||
Run from the repository root with a default provider and model:
|
||||
|
||||
```bash
|
||||
npm run eval -- --provider openai-codex --model gpt-5.4
|
||||
npm run eval -- --provider openai --model gpt-5.6-sol
|
||||
```
|
||||
|
||||
When invoked from a Pi Bash tool, the current session supplies `PI_PROVIDER` and `PI_MODEL`, so this is sufficient:
|
||||
The equivalent environment variables are:
|
||||
|
||||
```bash
|
||||
npm run eval
|
||||
PI_PROVIDER=openai PI_MODEL=gpt-5.6-sol npm run eval
|
||||
```
|
||||
|
||||
The runner requires both values and never falls back to another model. Additional arguments are forwarded to Vitest, for example:
|
||||
CLI values take precedence and become defaults for harnesses that do not select a model explicitly. Provider and model must be supplied together. The runner also allows no default when every executed harness configures its own model.
|
||||
Authentication comes from Pi's normal `ModelRuntime`, including Pi subscription credentials and provider API-key
|
||||
environment variables.
|
||||
|
||||
Additional arguments are forwarded to Vitest:
|
||||
|
||||
```bash
|
||||
npm run eval -- -t "capital of France"
|
||||
npm run eval -- src/extensions.eval.ts
|
||||
npm run eval -- -t "creates, reloads, and uses"
|
||||
```
|
||||
|
||||
Authentication is resolved by Pi's normal `ModelRuntime`. Subscription-backed providers such as `openai-codex` use credentials from the user's Pi configuration. API-backed providers use their standard environment variables, such as `OPENAI_API_KEY` for `openai`.
|
||||
Each invocation prints an ignored `.eval/` artifact directory. `runs.jsonl` indexes completed harness runs and their
|
||||
native Pi session JSONL attachments under `sessions/`. These files may contain prompts, responses, source code, and tool
|
||||
output.
|
||||
|
||||
## Writing evals
|
||||
|
||||
Follow [`vitest-evals`](https://github.com/getsentry/vitest-evals) for general suite, judge, assertion, and normalized
|
||||
trace guidance. Pi-specific evals use `createPiCodingAgentHarness(...)` from `src/pi-harness.ts`, with one harness bound
|
||||
to each `describeEval(...)` suite:
|
||||
|
||||
```ts
|
||||
import { expect } from "vitest";
|
||||
import { describeEval } from "vitest-evals";
|
||||
import { createPiCodingAgentHarness } from "./pi-harness.ts";
|
||||
|
||||
const harness = createPiCodingAgentHarness({ noTools: "all" });
|
||||
|
||||
describeEval("Pi smoke", { harness }, (it) => {
|
||||
it("answers a factual question", async ({ run }) => {
|
||||
const result = await run("What is the capital of France? Reply with only the city name.");
|
||||
expect(result.output).toBe("Paris");
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Configuring the Pi harness
|
||||
|
||||
`createPiCodingAgentHarness(...)` accepts:
|
||||
|
||||
- `name`: stable harness identity used by reports and comparisons.
|
||||
- `model`: optional `{ provider, id }` selection. It overrides the runner's default model.
|
||||
- `noTools`: Pi's tool-disable configuration.
|
||||
- `transformSystemPrompt`: transforms the complete default prompt before the eval starts.
|
||||
- `output`: transforms the final response and `AgentSession` into a JSON-safe domain result.
|
||||
|
||||
An explicitly selected model makes model-comparison harnesses independent of the runner default:
|
||||
|
||||
```ts
|
||||
const harness = createPiCodingAgentHarness({
|
||||
name: "claude-opus-4-6",
|
||||
model: { provider: "anthropic", id: "claude-opus-4-6" },
|
||||
});
|
||||
```
|
||||
|
||||
A run accepts either one prompt or a sequence of prompt and reload steps. Reload steps are useful when the preceding
|
||||
prompt creates or changes Pi resources:
|
||||
|
||||
```ts
|
||||
const result = await run([
|
||||
{ type: "prompt", content: "Create a Pi extension." },
|
||||
{ type: "reload" },
|
||||
{ type: "prompt", content: "Use the extension." },
|
||||
]);
|
||||
```
|
||||
|
||||
### Transforming harness output
|
||||
|
||||
Use `output` to expose scenario-specific, JSON-safe behavior without adding that behavior to the generic Pi adapter:
|
||||
|
||||
```ts
|
||||
const harness = createPiCodingAgentHarness({
|
||||
output: ({ response, session }) => ({
|
||||
response,
|
||||
activeTools: session.getActiveToolNames(),
|
||||
extensionErrors: session.resourceLoader.getExtensions().errors,
|
||||
}),
|
||||
});
|
||||
```
|
||||
|
||||
Assert application behavior on `result.output`. Assert model and tool traces on `result.session`, using
|
||||
`vitest-evals` helpers such as `toolCalls(...)`.
|
||||
|
||||
### Writing comparative eval sets
|
||||
|
||||
Use `evalHarnessTable(...)` with Vitest's native `describe.for(...)` to run the same inputs against multiple harnesses.
|
||||
Harnesses may differ by prompt, tools, skills, model, or any other Pi configuration:
|
||||
|
||||
```ts
|
||||
import { describe } from "vitest";
|
||||
import { createJudge, describeEval } from "vitest-evals";
|
||||
import { evalHarnessTable } from "./vitest-evals/harness-table.ts";
|
||||
|
||||
const TargetTaskJudge = createJudge<string, string>("TargetTaskJudge", ({ output }) => ({
|
||||
score: output === "expected result" ? 1 : 0,
|
||||
}));
|
||||
|
||||
const harnessTable = evalHarnessTable(
|
||||
"target skill effectiveness",
|
||||
[withoutTargetSkillHarness, withTargetSkillHarness],
|
||||
{
|
||||
repetitions: 6,
|
||||
seed: 42,
|
||||
},
|
||||
);
|
||||
|
||||
describe.for(harnessTable)("$name repetition $repetition", ({ harness }) => {
|
||||
describeEval("target skill effectiveness", { harness, judges: [TargetTaskJudge], judgeThreshold: null }, (it) => {
|
||||
it("completes the target task", async ({ run }) => {
|
||||
await run("Complete the target task.");
|
||||
});
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
Comparative suites should record correctness with deterministic or model-backed judges and set `judgeThreshold: null`.
|
||||
This keeps a low score as an observation instead of making the Vitest invocation fail. Use hard assertions only for
|
||||
suite invariants and infrastructure contracts. `expect.soft(...)` still fails the test and is not a scoring mechanism.
|
||||
|
||||
Harness names must be stable and unique within an eval set. The grouping key combines repetition and seed with a
|
||||
non-empty string `input.id` when available, otherwise with a SHA-256 hash of strict canonical JSON input. Execution order
|
||||
is seeded and randomized, while harness declaration order determines comparison direction. For each matched input and
|
||||
repetition, the reporter computes pass-rate lift from each run's recorded average judge score, treating a score of at
|
||||
least `1` as passing. The lift is the later-declared harness's pass rate minus the earlier-declared harness's pass rate,
|
||||
in percentage points. Missing judge scores are reported as incomplete observations. Tokens, latency, and estimated cost
|
||||
remain separate paired deltas; missing telemetry remains unavailable. Sets
|
||||
with more than two harnesses report every pair.
|
||||
|
||||
See the [`skill-eval-harness`](https://github.com/adewale/skill-eval-harness/) guidance for comparative-eval methodology,
|
||||
repetition strategy, trustworthy judges, and telemetry interpretation.
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@
|
|||
"type": "module",
|
||||
"scripts": {
|
||||
"clean": "shx rm -rf .eval",
|
||||
"eval": "node scripts/run-evals.mjs"
|
||||
"eval": "node scripts/run-evals.mjs",
|
||||
"test": "vitest run --config vitest.test.config.ts"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@earendil-works/pi-ai": "^0.82.1",
|
||||
|
|
|
|||
|
|
@ -1,9 +1,18 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { mkdirSync } from "node:fs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createRequire } from "node:module";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const artifactDirectory = process.env.PI_EVAL_ARTIFACT_DIR
|
||||
? resolve(packageRoot, process.env.PI_EVAL_ARTIFACT_DIR)
|
||||
: resolve(
|
||||
packageRoot,
|
||||
".eval",
|
||||
`${new Date().toISOString().replaceAll(":", "-")}_${randomUUID()}`,
|
||||
);
|
||||
const args = process.argv.slice(2);
|
||||
let provider;
|
||||
let model;
|
||||
|
|
@ -37,37 +46,47 @@ for (let index = 0; index < args.length; index += 1) {
|
|||
vitestArgs.push(arg);
|
||||
}
|
||||
|
||||
if (hasCliModelSelection && (!provider || !model)) {
|
||||
console.error("CLI model selection requires both --provider and --model.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
provider ??= process.env.PI_PROVIDER;
|
||||
model ??= process.env.PI_MODEL;
|
||||
|
||||
if (!provider || !model) {
|
||||
console.error(
|
||||
"No eval model selected. Pass --provider and --model, or set PI_PROVIDER and PI_MODEL.",
|
||||
);
|
||||
process.exit(1);
|
||||
provider = provider?.trim() || undefined;
|
||||
model = model?.trim() || undefined;
|
||||
if (hasCliModelSelection) {
|
||||
if (!provider || !model) {
|
||||
console.error("CLI model selection requires both --provider and --model.");
|
||||
process.exit(1);
|
||||
}
|
||||
} else {
|
||||
provider = process.env.PI_PROVIDER?.trim() || undefined;
|
||||
model = process.env.PI_MODEL?.trim() || undefined;
|
||||
if (Boolean(provider) !== Boolean(model)) {
|
||||
console.error("Default model selection requires both PI_PROVIDER and PI_MODEL.");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const vitestPackagePath = require.resolve("vitest/package.json");
|
||||
const vitestCliPath = resolve(dirname(vitestPackagePath), "vitest.mjs");
|
||||
|
||||
console.error(`[eval] provider=${provider} model=${model}`);
|
||||
mkdirSync(artifactDirectory, { recursive: true, mode: 0o700 });
|
||||
console.error(`[eval] default-model=${provider && model ? `${provider}/${model}` : "none"}`);
|
||||
console.error(`[eval] artifacts=${artifactDirectory}`);
|
||||
const childEnvironment = {
|
||||
...process.env,
|
||||
PI_EVAL_ARTIFACT_DIR: artifactDirectory,
|
||||
};
|
||||
if (provider && model) {
|
||||
childEnvironment.PI_PROVIDER = provider;
|
||||
childEnvironment.PI_MODEL = model;
|
||||
} else {
|
||||
delete childEnvironment.PI_PROVIDER;
|
||||
delete childEnvironment.PI_MODEL;
|
||||
}
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[vitestCliPath, "run", "--config", "vitest.config.ts", ...vitestArgs],
|
||||
{
|
||||
cwd: packageRoot,
|
||||
stdio: "inherit",
|
||||
env: {
|
||||
...process.env,
|
||||
PI_PROVIDER: provider,
|
||||
PI_MODEL: model,
|
||||
},
|
||||
env: childEnvironment,
|
||||
},
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,42 +1,139 @@
|
|||
import { expect } from "vitest";
|
||||
import { describeEval, toolCalls } from "vitest-evals";
|
||||
import { createPiCodingAgentHarness } from "./pi-harness.ts";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect } from "vitest";
|
||||
import { createJudge, describeEval } from "vitest-evals";
|
||||
import { createPiCodingAgentHarness, type PiCodingAgentInput } from "./pi-harness.ts";
|
||||
import { recordEvalExtensionSourceArtifact } from "./vitest-evals/artifacts.ts";
|
||||
import { evalHarnessTable } from "./vitest-evals/harness-table.ts";
|
||||
|
||||
type ExtensionAuthoringOutput = {
|
||||
response: string;
|
||||
systemPromptHasGuidelines: boolean;
|
||||
systemPromptHasPiDocs: boolean;
|
||||
extensionErrors: Array<{ path: string; error: string }>;
|
||||
loadedExtensions: Array<{ path: string; tools: string[] }>;
|
||||
extensionSource: string | null;
|
||||
};
|
||||
|
||||
function createExtensionAuthoringHarness(name: string, transformSystemPrompt?: (defaultPrompt: string) => string) {
|
||||
return createPiCodingAgentHarness({
|
||||
name,
|
||||
...(transformSystemPrompt ? { transformSystemPrompt } : {}),
|
||||
output: ({ response, session }) => {
|
||||
const extensions = session.resourceLoader.getExtensions();
|
||||
const extensionPath = join(session.sessionManager.getCwd(), ".pi", "extensions", "hello.ts");
|
||||
const extensionSource = existsSync(extensionPath) ? readFileSync(extensionPath, "utf8") : null;
|
||||
return {
|
||||
response,
|
||||
systemPromptHasGuidelines: session.systemPrompt.includes("\nGuidelines:\n"),
|
||||
systemPromptHasPiDocs: session.systemPrompt.includes("\nPi documentation (read only"),
|
||||
extensionErrors: extensions.errors,
|
||||
loadedExtensions: extensions.extensions.map(({ path, tools }) => ({
|
||||
path,
|
||||
tools: [...tools.keys()],
|
||||
})),
|
||||
extensionSource,
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function excludeGuidelinesAndDocumentation(defaultPrompt: string): string {
|
||||
const guidelinesStart = defaultPrompt.indexOf("\nGuidelines:\n");
|
||||
if (guidelinesStart === -1) throw new Error("Default Pi system prompt has no Guidelines section.");
|
||||
return defaultPrompt.slice(0, guidelinesStart);
|
||||
}
|
||||
|
||||
function prepareDefaultPromptOverride(defaultPrompt: string): string {
|
||||
const cwdStart = defaultPrompt.lastIndexOf("\nCurrent working directory: ");
|
||||
if (cwdStart === -1) throw new Error("Default Pi system prompt has no working-directory section.");
|
||||
return defaultPrompt.slice(0, cwdStart);
|
||||
}
|
||||
|
||||
const ExtensionAuthoringJudge = createJudge<PiCodingAgentInput, ExtensionAuthoringOutput>(
|
||||
"ExtensionAuthoringJudge",
|
||||
({ output, toolCalls }) => {
|
||||
const failures: string[] = [];
|
||||
if (output.extensionSource === null) {
|
||||
failures.push("generated extension source is unavailable");
|
||||
} else {
|
||||
const imports = Array.from(
|
||||
output.extensionSource.matchAll(/\b(?:from|import)\s+["']([^"']+)["']/g),
|
||||
(match) => match[1],
|
||||
);
|
||||
if (!imports.includes("@earendil-works/pi-coding-agent")) {
|
||||
failures.push("extension does not import the canonical @earendil-works/pi-coding-agent package");
|
||||
}
|
||||
if (imports.some((specifier) => specifier.startsWith("@mariozechner/"))) {
|
||||
failures.push("extension imports a legacy @mariozechner package");
|
||||
}
|
||||
if (imports.some((specifier) => specifier.startsWith("@sinclair/typebox"))) {
|
||||
failures.push('extension imports legacy "@sinclair/typebox" instead of "typebox"');
|
||||
}
|
||||
}
|
||||
if (output.extensionErrors.length > 0) failures.push("extension loader reported errors");
|
||||
if (!output.loadedExtensions.some(({ tools }) => tools.includes("hello"))) {
|
||||
failures.push('no loaded extension registered the "hello" tool');
|
||||
}
|
||||
if (
|
||||
!toolCalls.some(
|
||||
(call) =>
|
||||
call.name === "hello" &&
|
||||
call.status === "ok" &&
|
||||
call.arguments?.name === "Bob" &&
|
||||
call.result === "Hello, Bob!",
|
||||
)
|
||||
) {
|
||||
failures.push('no successful hello({ name: "Bob" }) call returned "Hello, Bob!"');
|
||||
}
|
||||
if (output.response !== "Hello, Bob!") failures.push('final response was not exactly "Hello, Bob!"');
|
||||
|
||||
const piExtensionsHarness = createPiCodingAgentHarness({
|
||||
name: "pi-coding-agent-extensions",
|
||||
output: ({ response, session }) => {
|
||||
return {
|
||||
response,
|
||||
extensionPaths: session.extensionRunner.getExtensionPaths(),
|
||||
toolDefinitions: session.getAllTools().map(({ name, description }) => ({ name, description })),
|
||||
score: failures.length === 0 ? 1 : 0,
|
||||
metadata: {
|
||||
rationale: failures.length === 0 ? "Extension authoring workflow completed." : failures.join("; "),
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
);
|
||||
|
||||
describeEval("Pi extensions", { harness: piExtensionsHarness }, (it) => {
|
||||
it("creates, reloads, and uses a hello extension", async ({ run }) => {
|
||||
const result = await run([
|
||||
{
|
||||
type: "prompt",
|
||||
content:
|
||||
"Create a Pi extension with a hello tool that takes a name and returns a greeting. For example, passing Bob should return `Hello, Bob!`.",
|
||||
},
|
||||
{ type: "reload" },
|
||||
{
|
||||
type: "prompt",
|
||||
content: "Use the hello tool to greet Bob.",
|
||||
},
|
||||
]);
|
||||
const extensionHarnessTable = evalHarnessTable(
|
||||
"Pi extension authoring system prompt",
|
||||
[
|
||||
createExtensionAuthoringHarness("system-prompt-without-docs", excludeGuidelinesAndDocumentation),
|
||||
createExtensionAuthoringHarness("default-system-prompt", prepareDefaultPromptOverride),
|
||||
],
|
||||
{ seed: 42 },
|
||||
);
|
||||
|
||||
expect(result.output.response).toBe("Hello, Bob!");
|
||||
expect(result.output.extensionPaths).toHaveLength(1);
|
||||
expect(result.output.toolDefinitions).toContainEqual(expect.objectContaining({ name: "hello" }));
|
||||
expect(result.errors).toEqual([]);
|
||||
expect(toolCalls(result.session)).toContainEqual({
|
||||
name: "hello",
|
||||
arguments: { name: "Bob" },
|
||||
status: "ok",
|
||||
result: "Hello, Bob!",
|
||||
});
|
||||
});
|
||||
describe.for(extensionHarnessTable)("$name", ({ harness }) => {
|
||||
describeEval(
|
||||
"Pi extension authoring system prompt",
|
||||
{ harness, judges: [ExtensionAuthoringJudge], judgeThreshold: null },
|
||||
(it) => {
|
||||
it("creates, reloads, and uses a hello extension", async ({ run, task }) => {
|
||||
const result = await run([
|
||||
{
|
||||
type: "prompt",
|
||||
content:
|
||||
"Create a Pi extension with a hello tool that takes a name and returns a greeting. For example, passing Bob should return `Hello, Bob!`.",
|
||||
},
|
||||
{ type: "reload" },
|
||||
{
|
||||
type: "prompt",
|
||||
content:
|
||||
"Use the hello tool to greet Bob. Respond with exactly the tool's greeting and nothing else.",
|
||||
},
|
||||
]);
|
||||
if (result.output.extensionSource !== null) {
|
||||
const runId = result.artifacts?.runId;
|
||||
if (typeof runId !== "string") throw new Error("Pi eval run did not record a run ID.");
|
||||
await recordEvalExtensionSourceArtifact(task, runId, result.output.extensionSource);
|
||||
}
|
||||
const expectsFullPrompt = harness.name === "default-system-prompt";
|
||||
expect(result.output.systemPromptHasGuidelines).toBe(expectsFullPrompt);
|
||||
expect(result.output.systemPromptHasPiDocs).toBe(expectsFullPrompt);
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
import assert from "node:assert";
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdir, mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { contentText } from "@earendil-works/pi-ai";
|
||||
import {
|
||||
type AgentSession,
|
||||
|
|
@ -15,29 +17,43 @@ import {
|
|||
import {
|
||||
createHarness,
|
||||
type Harness,
|
||||
type HarnessContext,
|
||||
type JsonValue,
|
||||
normalizeRecord,
|
||||
type SimpleHarnessResult,
|
||||
type TranscriptEvent,
|
||||
toJsonValue,
|
||||
} from "vitest-evals/harness";
|
||||
import { getCurrentEvalTest, recordEvalFileArtifact } from "./vitest-evals/artifacts.ts";
|
||||
|
||||
type PiCodingAgentInput = string | Array<{ type: "prompt"; content: string } | { type: "reload" }>;
|
||||
export type PiCodingAgentInput = string | Array<{ type: "prompt"; content: string } | { type: "reload" }>;
|
||||
|
||||
export type PiCodingAgentModelSelection = {
|
||||
provider: string;
|
||||
id: string;
|
||||
};
|
||||
|
||||
type PiCodingAgentHarnessOptions = {
|
||||
name?: string;
|
||||
model?: PiCodingAgentModelSelection;
|
||||
noTools?: CreateAgentSessionOptions["noTools"];
|
||||
transformSystemPrompt?: (defaultPrompt: string) => string;
|
||||
};
|
||||
|
||||
type PiCodingAgentHarnessWithOutput<TOutput extends JsonValue> = PiCodingAgentHarnessOptions & {
|
||||
output: (args: { response: string; session: AgentSession }) => TOutput | Promise<TOutput>;
|
||||
};
|
||||
|
||||
function getRequiredModelSelection(): { provider: string; model: string } {
|
||||
const provider = process.env.PI_PROVIDER?.trim();
|
||||
const model = process.env.PI_MODEL?.trim();
|
||||
if (!provider || !model) throw new Error("PI_PROVIDER and PI_MODEL must both be set for eval runs.");
|
||||
return { provider, model };
|
||||
export function resolveModelSelection(
|
||||
explicitModel: PiCodingAgentModelSelection | undefined,
|
||||
environment: { PI_PROVIDER?: string; PI_MODEL?: string } = process.env,
|
||||
): PiCodingAgentModelSelection {
|
||||
const provider = (explicitModel?.provider ?? environment.PI_PROVIDER)?.trim();
|
||||
const id = (explicitModel?.id ?? environment.PI_MODEL)?.trim();
|
||||
if (!provider || !id) {
|
||||
throw new Error("Select a harness model explicitly or set both PI_PROVIDER and PI_MODEL as defaults.");
|
||||
}
|
||||
return { provider, id };
|
||||
}
|
||||
|
||||
function toTranscriptEvents(messages: AgentSession["messages"]): TranscriptEvent[] {
|
||||
|
|
@ -94,17 +110,23 @@ async function promptAgent(session: AgentSession, input: string, signal: AbortSi
|
|||
async function runPiCodingAgent<TOutput extends JsonValue>(
|
||||
input: PiCodingAgentInput,
|
||||
signal: AbortSignal | undefined,
|
||||
setArtifact: HarnessContext["setArtifact"],
|
||||
options: PiCodingAgentHarnessOptions | PiCodingAgentHarnessWithOutput<TOutput>,
|
||||
): Promise<SimpleHarnessResult<string | TOutput>> {
|
||||
const startedAt = performance.now();
|
||||
const artifactDirectory = process.env.PI_EVAL_ARTIFACT_DIR?.trim();
|
||||
const evalTest = artifactDirectory ? getCurrentEvalTest() : undefined;
|
||||
signal?.throwIfAborted();
|
||||
const selection = getRequiredModelSelection();
|
||||
const selection = resolveModelSelection(options.model);
|
||||
const modelRuntime = await ModelRuntime.create();
|
||||
const model = modelRuntime.getModel(selection.provider, selection.model);
|
||||
if (!model) throw new Error(`Eval model not found: ${selection.provider}/${selection.model}`);
|
||||
const model = modelRuntime.getModel(selection.provider, selection.id);
|
||||
if (!model) throw new Error(`Eval model not found: ${selection.provider}/${selection.id}`);
|
||||
|
||||
const root = await mkdtemp(join(tmpdir(), "pi-eval-"));
|
||||
const cwd = join(root, "workspace");
|
||||
const agentDir = join(root, "agent");
|
||||
let transformedSystemPrompt: string | undefined;
|
||||
let sessionManager: SessionManager | undefined;
|
||||
let session: AgentSession | undefined;
|
||||
let outcome: { success: true; result: SimpleHarnessResult<string | TOutput> } | { success: false; error: unknown };
|
||||
try {
|
||||
|
|
@ -114,12 +136,17 @@ async function runPiCodingAgent<TOutput extends JsonValue>(
|
|||
agentDir,
|
||||
modelRuntime,
|
||||
settingsManager: SettingsManager.inMemory(),
|
||||
...(options.transformSystemPrompt
|
||||
? { resourceLoaderOptions: { systemPromptOverride: () => transformedSystemPrompt } }
|
||||
: {}),
|
||||
});
|
||||
signal?.throwIfAborted();
|
||||
sessionManager = SessionManager.create(cwd, join(root, "sessions"));
|
||||
setArtifact("runId", sessionManager.getSessionId());
|
||||
session = (
|
||||
await createAgentSessionFromServices({
|
||||
services,
|
||||
sessionManager: SessionManager.inMemory(cwd),
|
||||
sessionManager,
|
||||
model,
|
||||
thinkingLevel: "off",
|
||||
noTools: options.noTools,
|
||||
|
|
@ -127,7 +154,15 @@ async function runPiCodingAgent<TOutput extends JsonValue>(
|
|||
).session;
|
||||
|
||||
const evalSession = session;
|
||||
const abort = () => void evalSession.abort();
|
||||
if (options.transformSystemPrompt) {
|
||||
transformedSystemPrompt = options.transformSystemPrompt(evalSession.systemPrompt);
|
||||
if (!transformedSystemPrompt.trim()) throw new Error("Transformed eval system prompt must not be empty.");
|
||||
await evalSession.reload();
|
||||
}
|
||||
let abortPromise: Promise<void> | undefined;
|
||||
const abort = () => {
|
||||
abortPromise ??= evalSession.abort();
|
||||
};
|
||||
signal?.addEventListener("abort", abort, { once: true });
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
|
|
@ -148,6 +183,9 @@ async function runPiCodingAgent<TOutput extends JsonValue>(
|
|||
if (response === undefined) throw new Error("Pi eval input must include at least one prompt step.");
|
||||
const output = "output" in options ? await options.output({ response, session: evalSession }) : response;
|
||||
const stats = evalSession.getSessionStats();
|
||||
const hasPricing = [model.cost, ...(model.cost.tiers ?? [])].some(
|
||||
({ input, output, cacheRead, cacheWrite }) => input > 0 || output > 0 || cacheRead > 0 || cacheWrite > 0,
|
||||
);
|
||||
outcome = {
|
||||
success: true,
|
||||
result: {
|
||||
|
|
@ -160,17 +198,36 @@ async function runPiCodingAgent<TOutput extends JsonValue>(
|
|||
outputTokens: stats.tokens.output,
|
||||
totalTokens: stats.tokens.total,
|
||||
toolCalls: stats.toolCalls,
|
||||
metadata: {
|
||||
cacheReadTokens: stats.tokens.cacheRead,
|
||||
cacheWriteTokens: stats.tokens.cacheWrite,
|
||||
...(hasPricing ? { estimatedCostUsd: stats.cost } : {}),
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
signal?.removeEventListener("abort", abort);
|
||||
if (abortPromise) await abortPromise;
|
||||
}
|
||||
} catch (error) {
|
||||
outcome = { success: false, error };
|
||||
}
|
||||
|
||||
const cleanupErrors: unknown[] = [];
|
||||
if (sessionManager && artifactDirectory && evalTest) {
|
||||
try {
|
||||
const sessionPath = sessionManager.getSessionFile();
|
||||
if (sessionPath && existsSync(sessionPath)) {
|
||||
await recordEvalFileArtifact(evalTest, sessionManager.getSessionId(), {
|
||||
name: "session",
|
||||
path: sessionPath,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
cleanupErrors.push(error);
|
||||
}
|
||||
}
|
||||
try {
|
||||
session?.dispose();
|
||||
} catch (error) {
|
||||
|
|
@ -188,7 +245,10 @@ async function runPiCodingAgent<TOutput extends JsonValue>(
|
|||
}
|
||||
if (cleanupErrors.length === 1) throw cleanupErrors[0];
|
||||
if (cleanupErrors.length > 1) throw new AggregateError(cleanupErrors, "Agent cleanup failed.");
|
||||
return outcome.result;
|
||||
return {
|
||||
...outcome.result,
|
||||
timings: { totalMs: performance.now() - startedAt },
|
||||
};
|
||||
}
|
||||
|
||||
export function createPiCodingAgentHarness<TOutput extends JsonValue>(
|
||||
|
|
@ -200,6 +260,6 @@ export function createPiCodingAgentHarness<TOutput extends JsonValue>(
|
|||
) {
|
||||
return createHarness<PiCodingAgentInput, string | TOutput>({
|
||||
name: options.name ?? "pi-coding-agent",
|
||||
run: ({ input, signal }) => runPiCodingAgent(input, signal, options),
|
||||
run: ({ input, signal, setArtifact }) => runPiCodingAgent(input, signal, setArtifact, options),
|
||||
});
|
||||
}
|
||||
|
|
|
|||
111
packages/evals/src/vitest-evals/artifacts.ts
Normal file
111
packages/evals/src/vitest-evals/artifacts.ts
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
import { createHash } from "node:crypto";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { basename, isAbsolute, join, relative } from "node:path";
|
||||
import {
|
||||
type RunnerTestCase,
|
||||
recordArtifact,
|
||||
type TestArtifact,
|
||||
type TestArtifactBase,
|
||||
type TestAttachment,
|
||||
TestRunner,
|
||||
} from "vitest";
|
||||
|
||||
const evalFileArtifactKey = Symbol("pi-evals-file-artifact");
|
||||
const evalExtensionSourceArtifactKey = Symbol("pi-evals-extension-source-artifact");
|
||||
|
||||
export interface EvalFileAttachment extends TestAttachment {
|
||||
name: string;
|
||||
path: string;
|
||||
}
|
||||
|
||||
interface EvalExtensionSourceAttachment extends TestAttachment {
|
||||
name: string;
|
||||
contentType: "text/typescript";
|
||||
body: string;
|
||||
bodyEncoding: "utf-8";
|
||||
}
|
||||
|
||||
interface EvalFileArtifact extends TestArtifactBase {
|
||||
type: "@earendil-works/pi-evals:file";
|
||||
runId: string;
|
||||
attachments: EvalFileAttachment[];
|
||||
}
|
||||
|
||||
interface EvalExtensionSourceArtifact extends TestArtifactBase {
|
||||
type: "@earendil-works/pi-evals:extension-source";
|
||||
runId: string;
|
||||
attachments: [EvalExtensionSourceAttachment] | [];
|
||||
}
|
||||
|
||||
declare module "vitest" {
|
||||
interface TestArtifactRegistry {
|
||||
[evalFileArtifactKey]: EvalFileArtifact;
|
||||
[evalExtensionSourceArtifactKey]: EvalExtensionSourceArtifact;
|
||||
}
|
||||
}
|
||||
|
||||
export function getCurrentEvalTest(): NonNullable<ReturnType<typeof TestRunner.getCurrentTest>> {
|
||||
const test = TestRunner.getCurrentTest();
|
||||
if (!test) throw new Error("Cannot record an eval artifact outside a running Vitest test.");
|
||||
return test;
|
||||
}
|
||||
|
||||
export async function recordEvalFileArtifact(
|
||||
test: NonNullable<ReturnType<typeof TestRunner.getCurrentTest>>,
|
||||
runId: string,
|
||||
attachment: EvalFileAttachment,
|
||||
): Promise<void> {
|
||||
await recordArtifact(test, {
|
||||
type: "@earendil-works/pi-evals:file",
|
||||
runId,
|
||||
attachments: [attachment],
|
||||
});
|
||||
}
|
||||
|
||||
export async function recordEvalExtensionSourceArtifact(
|
||||
task: Readonly<RunnerTestCase>,
|
||||
runId: string,
|
||||
source: string,
|
||||
): Promise<void> {
|
||||
await recordArtifact(task, {
|
||||
type: "@earendil-works/pi-evals:extension-source",
|
||||
runId,
|
||||
attachments: [
|
||||
{
|
||||
name: "hello.ts",
|
||||
contentType: "text/typescript",
|
||||
body: source,
|
||||
bodyEncoding: "utf-8",
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
export async function persistEvalArtifactReferences(
|
||||
artifacts: ReadonlyArray<TestArtifact>,
|
||||
runId: string,
|
||||
artifactDirectory: string,
|
||||
): Promise<Array<{ name: string; path: string }>> {
|
||||
const references: Array<{ name: string; path: string }> = [];
|
||||
for (const artifact of artifacts) {
|
||||
if (artifact.type === "@earendil-works/pi-evals:file" && artifact.runId === runId) {
|
||||
for (const attachment of artifact.attachments) {
|
||||
references.push({
|
||||
name: attachment.name,
|
||||
path: isAbsolute(attachment.path) ? relative(artifactDirectory, attachment.path) : attachment.path,
|
||||
});
|
||||
}
|
||||
} else if (artifact.type === "@earendil-works/pi-evals:extension-source" && artifact.runId === runId) {
|
||||
for (const attachment of artifact.attachments) {
|
||||
const name = basename(attachment.name);
|
||||
if (name !== attachment.name) throw new TypeError(`Invalid eval artifact name: ${attachment.name}`);
|
||||
const directory = join(artifactDirectory, "sources", createHash("sha256").update(runId).digest("hex"));
|
||||
await mkdir(directory, { recursive: true, mode: 0o700 });
|
||||
const path = join(directory, name);
|
||||
await writeFile(path, attachment.body, { encoding: "utf8", mode: 0o600 });
|
||||
references.push({ name, path: relative(artifactDirectory, path) });
|
||||
}
|
||||
}
|
||||
}
|
||||
return references;
|
||||
}
|
||||
199
packages/evals/src/vitest-evals/harness-table.ts
Normal file
199
packages/evals/src/vitest-evals/harness-table.ts
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { attachHarnessRunToError, getHarnessRunFromError, type Harness, type JsonValue } from "vitest-evals/harness";
|
||||
|
||||
export const EVAL_HARNESS_ITERATION_ARTIFACT = "vitestEvalsHarnessIteration";
|
||||
|
||||
export type EvalHarnessIterationArtifact = {
|
||||
schemaVersion: 1;
|
||||
evalSet: string;
|
||||
groupKey: string;
|
||||
harness: string;
|
||||
harnesses: string[];
|
||||
repetition: number;
|
||||
seed: number;
|
||||
plannedOrder: number;
|
||||
};
|
||||
|
||||
export type EvalHarnessTableRow<TInput, TOutput extends JsonValue | undefined> = {
|
||||
harness: Harness<TInput, TOutput>;
|
||||
name: string;
|
||||
repetition: number;
|
||||
seed: number;
|
||||
plannedOrder: number;
|
||||
};
|
||||
|
||||
export type EvalHarnessTableOptions = {
|
||||
repetitions?: number;
|
||||
seed?: number;
|
||||
};
|
||||
|
||||
type EvalHarnessIterationPlan = Omit<EvalHarnessIterationArtifact, "groupKey">;
|
||||
|
||||
export function parseEvalHarnessIterationArtifact(
|
||||
value: JsonValue | undefined,
|
||||
): EvalHarnessIterationArtifact | undefined {
|
||||
if (value === null || value === undefined || typeof value !== "object" || Array.isArray(value)) return undefined;
|
||||
const { schemaVersion, evalSet, groupKey, harness, harnesses, repetition, seed, plannedOrder } = value;
|
||||
if (
|
||||
schemaVersion !== 1 ||
|
||||
typeof evalSet !== "string" ||
|
||||
typeof groupKey !== "string" ||
|
||||
typeof harness !== "string" ||
|
||||
!Array.isArray(harnesses) ||
|
||||
!harnesses.every((name): name is string => typeof name === "string") ||
|
||||
typeof repetition !== "number" ||
|
||||
typeof seed !== "number" ||
|
||||
typeof plannedOrder !== "number"
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return { schemaVersion, evalSet, groupKey, harness, harnesses, repetition, seed, plannedOrder };
|
||||
}
|
||||
|
||||
function createSeededRandom(seed: number): () => number {
|
||||
let state = seed >>> 0;
|
||||
return () => {
|
||||
state += 0x6d2b79f5;
|
||||
let value = state;
|
||||
value = Math.imul(value ^ (value >>> 15), value | 1);
|
||||
value ^= value + Math.imul(value ^ (value >>> 7), value | 61);
|
||||
return ((value ^ (value >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
function serializeString(value: string): string {
|
||||
const serialized = JSON.stringify(value);
|
||||
if (serialized === undefined) throw new TypeError("Eval input must be JSON-serializable.");
|
||||
return serialized;
|
||||
}
|
||||
|
||||
function canonicalJson(value: unknown, ancestors: WeakSet<object>): string {
|
||||
if (value === null) return "null";
|
||||
if (typeof value === "string") return serializeString(value);
|
||||
if (typeof value === "boolean") return String(value);
|
||||
if (typeof value === "number") {
|
||||
if (!Number.isFinite(value)) throw new TypeError("Eval input must contain only finite numbers.");
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
if (typeof value !== "object") throw new TypeError("Eval input must be JSON-serializable.");
|
||||
if (ancestors.has(value)) throw new TypeError("Eval input must not contain circular references.");
|
||||
|
||||
ancestors.add(value);
|
||||
try {
|
||||
if (Array.isArray(value)) return `[${Array.from(value, (item) => canonicalJson(item, ancestors)).join(",")}]`;
|
||||
const prototype = Object.getPrototypeOf(value);
|
||||
if (prototype !== Object.prototype && prototype !== null) {
|
||||
throw new TypeError("Eval input must contain only plain objects and arrays.");
|
||||
}
|
||||
return `{${Object.entries(value)
|
||||
.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
|
||||
.map(([key, item]) => `${serializeString(key)}:${canonicalJson(item, ancestors)}`)
|
||||
.join(",")}}`;
|
||||
} finally {
|
||||
ancestors.delete(value);
|
||||
}
|
||||
}
|
||||
|
||||
function deriveInputKey(input: unknown): string {
|
||||
if (typeof input === "object" && input !== null && !Array.isArray(input) && "id" in input) {
|
||||
const id = input.id;
|
||||
if (typeof id === "string" && id.trim()) return id.trim();
|
||||
}
|
||||
return createHash("sha256").update(canonicalJson(input, new WeakSet())).digest("hex");
|
||||
}
|
||||
|
||||
export function deriveEvalGroupKey(input: unknown, repetition: number, seed: number): string {
|
||||
return JSON.stringify([deriveInputKey(input), repetition, seed]);
|
||||
}
|
||||
|
||||
function validateOptions<TInput, TOutput extends JsonValue | undefined>(
|
||||
evalSet: string,
|
||||
harnesses: readonly Harness<TInput, TOutput>[],
|
||||
repetitions: number,
|
||||
seed: number,
|
||||
): void {
|
||||
if (!evalSet.trim()) throw new TypeError("evalSet must not be empty.");
|
||||
if (harnesses.length < 2) throw new TypeError("At least two harnesses are required.");
|
||||
const names = new Set(harnesses.map((harness) => harness.name));
|
||||
if (names.size !== harnesses.length) throw new TypeError("Harness names must be unique within an eval set.");
|
||||
if (!Number.isSafeInteger(repetitions) || repetitions < 1) {
|
||||
throw new TypeError("repetitions must be a positive integer.");
|
||||
}
|
||||
if (!Number.isSafeInteger(seed) || seed < 0 || seed > 0xffffffff) {
|
||||
throw new TypeError("seed must be an unsigned 32-bit integer.");
|
||||
}
|
||||
}
|
||||
|
||||
function withIterationArtifact<TInput, TOutput extends JsonValue | undefined>(
|
||||
harness: Harness<TInput, TOutput>,
|
||||
plan: EvalHarnessIterationPlan,
|
||||
): Harness<TInput, TOutput> {
|
||||
return {
|
||||
name: harness.name,
|
||||
async run(input, context) {
|
||||
const groupKey = deriveEvalGroupKey(input, plan.repetition, plan.seed);
|
||||
const artifact: EvalHarnessIterationArtifact = { ...plan, groupKey };
|
||||
context.setArtifact(EVAL_HARNESS_ITERATION_ARTIFACT, artifact);
|
||||
try {
|
||||
const run = await harness.run(input, context);
|
||||
run.artifacts = { ...context.artifacts, ...run.artifacts, [EVAL_HARNESS_ITERATION_ARTIFACT]: artifact };
|
||||
return run;
|
||||
} catch (error) {
|
||||
const partialRun = getHarnessRunFromError(error);
|
||||
if (partialRun) {
|
||||
partialRun.artifacts = {
|
||||
...context.artifacts,
|
||||
...partialRun.artifacts,
|
||||
[EVAL_HARNESS_ITERATION_ARTIFACT]: artifact,
|
||||
};
|
||||
throw attachHarnessRunToError(error, partialRun);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function evalHarnessTable<TInput, TOutput extends JsonValue | undefined>(
|
||||
evalSet: string,
|
||||
harnesses: readonly Harness<TInput, TOutput>[],
|
||||
options: EvalHarnessTableOptions,
|
||||
): EvalHarnessTableRow<TInput, TOutput>[] {
|
||||
const repetitions = options.repetitions ?? 1;
|
||||
const seed = options.seed ?? randomBytes(4).readUInt32LE();
|
||||
validateOptions(evalSet, harnesses, repetitions, seed);
|
||||
|
||||
const random = createSeededRandom(seed);
|
||||
const rows: EvalHarnessTableRow<TInput, TOutput>[] = [];
|
||||
let plannedOrder = 1;
|
||||
for (let repetition = 1; repetition <= repetitions; repetition += 1) {
|
||||
const randomizedHarnesses = [...harnesses];
|
||||
for (let index = randomizedHarnesses.length - 1; index > 0; index -= 1) {
|
||||
const target = Math.floor(random() * (index + 1));
|
||||
[randomizedHarnesses[index], randomizedHarnesses[target]] = [
|
||||
randomizedHarnesses[target],
|
||||
randomizedHarnesses[index],
|
||||
];
|
||||
}
|
||||
for (const harness of randomizedHarnesses) {
|
||||
const plan: EvalHarnessIterationPlan = {
|
||||
schemaVersion: 1,
|
||||
evalSet,
|
||||
harness: harness.name,
|
||||
harnesses: harnesses.map(({ name }) => name),
|
||||
repetition,
|
||||
seed,
|
||||
plannedOrder,
|
||||
};
|
||||
rows.push({
|
||||
harness: withIterationArtifact(harness, plan),
|
||||
name: harness.name,
|
||||
repetition,
|
||||
seed,
|
||||
plannedOrder,
|
||||
});
|
||||
plannedOrder += 1;
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
121
packages/evals/src/vitest-evals/reporter.ts
Normal file
121
packages/evals/src/vitest-evals/reporter.ts
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { appendFile, mkdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import type { Reporter, SerializedError, TestCase, TestModule, TestRunEndReason, Vitest } from "vitest/node";
|
||||
import { isHarnessRun } from "vitest-evals/harness";
|
||||
import { persistEvalArtifactReferences } from "./artifacts.ts";
|
||||
import { EVAL_HARNESS_ITERATION_ARTIFACT, parseEvalHarnessIterationArtifact } from "./harness-table.ts";
|
||||
import { formatHarnessComparisonReport, type HarnessObservation, summarizeHarnessComparisons } from "./summary.ts";
|
||||
|
||||
function readFiniteNumber(value: unknown): number | undefined {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
|
||||
function readStatus(state: string): "passed" | "failed" | "skipped" | "pending" {
|
||||
if (state === "passed" || state === "failed" || state === "skipped") return state;
|
||||
return "pending";
|
||||
}
|
||||
|
||||
async function appendHarnessRunReport(test: TestCase): Promise<void> {
|
||||
const artifactDirectory = process.env.PI_EVAL_ARTIFACT_DIR?.trim();
|
||||
if (!artifactDirectory) return;
|
||||
const harness = test.meta().harness;
|
||||
if (!harness || !isHarnessRun(harness.run)) return;
|
||||
|
||||
const run = harness.run;
|
||||
const artifactRunId = run.artifacts?.runId;
|
||||
const runId = typeof artifactRunId === "string" ? artifactRunId : randomUUID();
|
||||
const metadata = Object.fromEntries(Object.entries(run.artifacts ?? {}).filter(([name]) => name !== "runId"));
|
||||
const record = {
|
||||
schemaVersion: 1,
|
||||
runId,
|
||||
test: {
|
||||
id: test.id,
|
||||
file: test.module.relativeModuleId,
|
||||
name: test.name,
|
||||
fullName: test.fullName,
|
||||
status: readStatus(test.result().state),
|
||||
},
|
||||
harness: harness.name,
|
||||
usage: run.usage,
|
||||
...(run.timings ? { timings: run.timings } : {}),
|
||||
...(run.errors.length > 0 ? { errors: run.errors } : {}),
|
||||
artifacts: await persistEvalArtifactReferences(test.artifacts(), runId, artifactDirectory),
|
||||
...(Object.keys(metadata).length > 0 ? { metadata } : {}),
|
||||
};
|
||||
await mkdir(artifactDirectory, { recursive: true, mode: 0o700 });
|
||||
await appendFile(join(artifactDirectory, "runs.jsonl"), `${JSON.stringify(record)}\n`, {
|
||||
encoding: "utf8",
|
||||
flag: "a",
|
||||
mode: 0o600,
|
||||
});
|
||||
}
|
||||
|
||||
export function collectHarnessObservations(modules: ReadonlyArray<TestModule>): HarnessObservation[] {
|
||||
const observations: HarnessObservation[] = [];
|
||||
for (const module of modules) {
|
||||
for (const test of module.children.allTests()) {
|
||||
const harness = test.meta().harness;
|
||||
if (!harness || !isHarnessRun(harness.run)) continue;
|
||||
const run = harness.run;
|
||||
const iteration = parseEvalHarnessIterationArtifact(run.artifacts?.[EVAL_HARNESS_ITERATION_ARTIFACT]);
|
||||
if (!iteration) continue;
|
||||
const score = readFiniteNumber(test.meta().eval?.avgScore);
|
||||
const estimatedCostUsd = readFiniteNumber(run.usage.metadata?.estimatedCostUsd);
|
||||
const observation = {
|
||||
evalSet: iteration.evalSet,
|
||||
groupKey: iteration.groupKey,
|
||||
testName: test.name,
|
||||
file: module.relativeModuleId,
|
||||
harness: iteration.harness,
|
||||
harnesses: iteration.harnesses,
|
||||
repetition: iteration.repetition,
|
||||
seed: iteration.seed,
|
||||
...(run.usage.totalTokens === undefined ? {} : { totalTokens: run.usage.totalTokens }),
|
||||
...(run.timings?.totalMs === undefined ? {} : { totalMs: run.timings.totalMs }),
|
||||
...(estimatedCostUsd === undefined ? {} : { estimatedCostUsd }),
|
||||
};
|
||||
if (run.errors.length > 0) observations.push({ ...observation, outcome: "errored" });
|
||||
else if (score !== undefined) observations.push({ ...observation, outcome: "scored", score });
|
||||
else {
|
||||
const state = test.result().state;
|
||||
const outcome =
|
||||
state === "skipped"
|
||||
? "skipped"
|
||||
: state === "passed"
|
||||
? "unscored"
|
||||
: state === "failed"
|
||||
? "errored"
|
||||
: "pending";
|
||||
observations.push({ ...observation, outcome });
|
||||
}
|
||||
}
|
||||
}
|
||||
return observations;
|
||||
}
|
||||
|
||||
export default class EvalHarnessReporter implements Reporter {
|
||||
private vitest: Vitest | undefined;
|
||||
|
||||
onInit(vitest: Vitest): void {
|
||||
this.vitest = vitest;
|
||||
}
|
||||
|
||||
async onTestCaseResult(test: TestCase): Promise<void> {
|
||||
await appendHarnessRunReport(test);
|
||||
}
|
||||
|
||||
onTestRunEnd(
|
||||
modules: ReadonlyArray<TestModule>,
|
||||
_errors: ReadonlyArray<SerializedError>,
|
||||
reason: TestRunEndReason,
|
||||
): void {
|
||||
if (reason === "interrupted") {
|
||||
this.vitest?.logger.log("\nHarness comparisons unavailable: test run interrupted.");
|
||||
return;
|
||||
}
|
||||
const report = summarizeHarnessComparisons(collectHarnessObservations(modules));
|
||||
const formatted = formatHarnessComparisonReport(report);
|
||||
if (formatted) this.vitest?.logger.log(`\n${formatted}`);
|
||||
}
|
||||
}
|
||||
413
packages/evals/src/vitest-evals/summary.ts
Normal file
413
packages/evals/src/vitest-evals/summary.ts
Normal file
|
|
@ -0,0 +1,413 @@
|
|||
export type HarnessObservationOutcome = "scored" | "unscored" | "skipped" | "pending" | "errored";
|
||||
|
||||
type HarnessObservationBase = {
|
||||
evalSet: string;
|
||||
groupKey: string;
|
||||
testName: string;
|
||||
file: string;
|
||||
harness: string;
|
||||
harnesses: string[];
|
||||
repetition: number;
|
||||
seed: number;
|
||||
totalTokens?: number;
|
||||
totalMs?: number;
|
||||
estimatedCostUsd?: number;
|
||||
};
|
||||
|
||||
export type HarnessObservation = HarnessObservationBase &
|
||||
({ outcome: "scored"; score: number } | { outcome: Exclude<HarnessObservationOutcome, "scored">; score?: never });
|
||||
|
||||
export type PairedMetricSummary = {
|
||||
totalPairs: number;
|
||||
eligiblePairs: number;
|
||||
firstMean: number | null;
|
||||
secondMean: number | null;
|
||||
meanDelta: number | null;
|
||||
};
|
||||
|
||||
export type CorrectnessLiftSummary = {
|
||||
totalPairs: number;
|
||||
eligiblePairs: number;
|
||||
firstPassRate: number | null;
|
||||
secondPassRate: number | null;
|
||||
lift: number | null;
|
||||
firstWins: number;
|
||||
secondWins: number;
|
||||
ties: number;
|
||||
};
|
||||
|
||||
export type HarnessPairComparison = {
|
||||
firstHarness: string;
|
||||
secondHarness: string;
|
||||
correctness: CorrectnessLiftSummary;
|
||||
totalTokens: PairedMetricSummary;
|
||||
totalMs: PairedMetricSummary;
|
||||
estimatedCostUsd: PairedMetricSummary;
|
||||
};
|
||||
|
||||
export type HarnessComparisonDiagnostic = {
|
||||
evalSet: string;
|
||||
groupKey: string;
|
||||
testName: string;
|
||||
file: string;
|
||||
repetition: number;
|
||||
seed: number;
|
||||
harness: string;
|
||||
reason: "missing-observation" | "duplicate-observation" | "harness-error" | "missing-score" | "unscorable-outcome";
|
||||
};
|
||||
|
||||
export type HarnessEvalSetReport = {
|
||||
evalSet: string;
|
||||
comparisons: HarnessPairComparison[];
|
||||
};
|
||||
|
||||
export type HarnessComparisonReport = {
|
||||
schemaVersion: 1;
|
||||
evalSets: HarnessEvalSetReport[];
|
||||
diagnostics: HarnessComparisonDiagnostic[];
|
||||
};
|
||||
|
||||
type HarnessDescriptor = {
|
||||
name: string;
|
||||
index: number;
|
||||
};
|
||||
|
||||
type ObservationGroup = {
|
||||
evalSet: string;
|
||||
groupKey: string;
|
||||
testName: string;
|
||||
file: string;
|
||||
repetition: number;
|
||||
seed: number;
|
||||
observationsByHarness: Map<string, HarnessObservation[]>;
|
||||
};
|
||||
|
||||
type EvalSetData = {
|
||||
harnessesByName: Map<string, HarnessDescriptor>;
|
||||
groupsByKey: Map<string, ObservationGroup>;
|
||||
};
|
||||
|
||||
type ObservationPair = {
|
||||
first: HarnessObservation;
|
||||
second: HarnessObservation;
|
||||
};
|
||||
|
||||
function getOrCreate<K, V extends object>(map: Map<K, V>, key: K, create: () => V): V {
|
||||
const existing = map.get(key);
|
||||
if (existing !== undefined) return existing;
|
||||
const value = create();
|
||||
map.set(key, value);
|
||||
return value;
|
||||
}
|
||||
|
||||
function mean(values: readonly number[]): number | null {
|
||||
return values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length;
|
||||
}
|
||||
|
||||
function preciseDifference(left: number, right: number): number {
|
||||
return Number((left - right).toPrecision(15));
|
||||
}
|
||||
|
||||
function isScorableObservation(
|
||||
observation: HarnessObservation,
|
||||
): observation is HarnessObservation & { outcome: "scored"; score: number } {
|
||||
return observation.outcome === "scored";
|
||||
}
|
||||
|
||||
function allPairs<T>(values: readonly T[]): Array<[T, T]> {
|
||||
const pairs: Array<[T, T]> = [];
|
||||
for (let first = 0; first < values.length; first += 1) {
|
||||
for (let second = first + 1; second < values.length; second += 1) {
|
||||
pairs.push([values[first], values[second]]);
|
||||
}
|
||||
}
|
||||
return pairs;
|
||||
}
|
||||
|
||||
function groupObservations(observations: readonly HarnessObservation[]): Map<string, EvalSetData> {
|
||||
const evalSets = new Map<string, EvalSetData>();
|
||||
for (const observation of observations) {
|
||||
const evalSet = getOrCreate(evalSets, observation.evalSet, () => ({
|
||||
harnessesByName: new Map(),
|
||||
groupsByKey: new Map(),
|
||||
}));
|
||||
|
||||
for (const [index, name] of observation.harnesses.entries()) {
|
||||
const existing = evalSet.harnessesByName.get(name);
|
||||
if (!existing || index < existing.index) evalSet.harnessesByName.set(name, { name, index });
|
||||
}
|
||||
|
||||
const group = getOrCreate(
|
||||
evalSet.groupsByKey,
|
||||
JSON.stringify([observation.file, observation.testName, observation.groupKey]),
|
||||
() => ({
|
||||
evalSet: observation.evalSet,
|
||||
groupKey: observation.groupKey,
|
||||
testName: observation.testName,
|
||||
file: observation.file,
|
||||
repetition: observation.repetition,
|
||||
seed: observation.seed,
|
||||
observationsByHarness: new Map(),
|
||||
}),
|
||||
);
|
||||
getOrCreate(group.observationsByHarness, observation.harness, (): HarnessObservation[] => []).push(observation);
|
||||
}
|
||||
return evalSets;
|
||||
}
|
||||
|
||||
function orderedHarnesses(evalSet: EvalSetData): HarnessDescriptor[] {
|
||||
return [...evalSet.harnessesByName.values()].sort(
|
||||
(left, right) => left.index - right.index || left.name.localeCompare(right.name),
|
||||
);
|
||||
}
|
||||
|
||||
function orderedGroups(evalSet: EvalSetData): ObservationGroup[] {
|
||||
return [...evalSet.groupsByKey.values()].sort(
|
||||
(left, right) =>
|
||||
left.groupKey.localeCompare(right.groupKey) || left.repetition - right.repetition || left.seed - right.seed,
|
||||
);
|
||||
}
|
||||
|
||||
function collectDiagnostics(
|
||||
harnesses: readonly HarnessDescriptor[],
|
||||
groups: readonly ObservationGroup[],
|
||||
): HarnessComparisonDiagnostic[] {
|
||||
const diagnostics: HarnessComparisonDiagnostic[] = [];
|
||||
for (const group of groups) {
|
||||
for (const { name: harness } of harnesses) {
|
||||
const observations = group.observationsByHarness.get(harness) ?? [];
|
||||
let reason: HarnessComparisonDiagnostic["reason"] | undefined;
|
||||
if (observations.length === 0) reason = "missing-observation";
|
||||
else if (observations.length > 1) reason = "duplicate-observation";
|
||||
else if (observations[0].outcome === "errored") reason = "harness-error";
|
||||
else if (observations[0].outcome === "unscored") {
|
||||
reason = "missing-score";
|
||||
} else if (observations[0].outcome !== "scored") {
|
||||
reason = "unscorable-outcome";
|
||||
}
|
||||
if (!reason) continue;
|
||||
diagnostics.push({
|
||||
evalSet: group.evalSet,
|
||||
groupKey: group.groupKey,
|
||||
testName: group.testName,
|
||||
file: group.file,
|
||||
repetition: group.repetition,
|
||||
seed: group.seed,
|
||||
harness,
|
||||
reason,
|
||||
});
|
||||
}
|
||||
}
|
||||
return diagnostics;
|
||||
}
|
||||
|
||||
function pairObservations(
|
||||
groups: readonly ObservationGroup[],
|
||||
firstHarness: string,
|
||||
secondHarness: string,
|
||||
): ObservationPair[] {
|
||||
const pairs: ObservationPair[] = [];
|
||||
for (const group of groups) {
|
||||
const first = group.observationsByHarness.get(firstHarness) ?? [];
|
||||
const second = group.observationsByHarness.get(secondHarness) ?? [];
|
||||
if (first.length === 1 && second.length === 1) pairs.push({ first: first[0], second: second[0] });
|
||||
}
|
||||
return pairs;
|
||||
}
|
||||
|
||||
function summarizeMetric(
|
||||
pairs: readonly ObservationPair[],
|
||||
select: (observation: HarnessObservation) => number | undefined,
|
||||
totalPairs: number,
|
||||
): PairedMetricSummary {
|
||||
const firstValues: number[] = [];
|
||||
const secondValues: number[] = [];
|
||||
for (const { first, second } of pairs) {
|
||||
if (!isScorableObservation(first) || !isScorableObservation(second)) continue;
|
||||
const firstValue = select(first);
|
||||
const secondValue = select(second);
|
||||
if (
|
||||
firstValue === undefined ||
|
||||
secondValue === undefined ||
|
||||
!Number.isFinite(firstValue) ||
|
||||
!Number.isFinite(secondValue)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
firstValues.push(firstValue);
|
||||
secondValues.push(secondValue);
|
||||
}
|
||||
|
||||
const firstMean = mean(firstValues);
|
||||
const secondMean = mean(secondValues);
|
||||
return {
|
||||
totalPairs,
|
||||
eligiblePairs: firstValues.length,
|
||||
firstMean,
|
||||
secondMean,
|
||||
meanDelta: firstMean === null || secondMean === null ? null : preciseDifference(secondMean, firstMean),
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeCorrectness(pairs: readonly ObservationPair[], totalPairs: number): CorrectnessLiftSummary {
|
||||
let eligiblePairs = 0;
|
||||
let firstPasses = 0;
|
||||
let secondPasses = 0;
|
||||
let firstWins = 0;
|
||||
let secondWins = 0;
|
||||
let ties = 0;
|
||||
|
||||
for (const { first, second } of pairs) {
|
||||
if (!isScorableObservation(first) || !isScorableObservation(second)) continue;
|
||||
eligiblePairs += 1;
|
||||
const firstPassed = first.score >= 1;
|
||||
const secondPassed = second.score >= 1;
|
||||
if (firstPassed) firstPasses += 1;
|
||||
if (secondPassed) secondPasses += 1;
|
||||
if (firstPassed === secondPassed) ties += 1;
|
||||
else if (firstPassed) firstWins += 1;
|
||||
else secondWins += 1;
|
||||
}
|
||||
|
||||
const firstPassRate = eligiblePairs === 0 ? null : firstPasses / eligiblePairs;
|
||||
const secondPassRate = eligiblePairs === 0 ? null : secondPasses / eligiblePairs;
|
||||
return {
|
||||
totalPairs,
|
||||
eligiblePairs,
|
||||
firstPassRate,
|
||||
secondPassRate,
|
||||
lift: firstPassRate === null || secondPassRate === null ? null : preciseDifference(secondPassRate, firstPassRate),
|
||||
firstWins,
|
||||
secondWins,
|
||||
ties,
|
||||
};
|
||||
}
|
||||
|
||||
function compareHarnesses(
|
||||
firstHarness: HarnessDescriptor,
|
||||
secondHarness: HarnessDescriptor,
|
||||
groups: readonly ObservationGroup[],
|
||||
): HarnessPairComparison {
|
||||
const pairs = pairObservations(groups, firstHarness.name, secondHarness.name);
|
||||
return {
|
||||
firstHarness: firstHarness.name,
|
||||
secondHarness: secondHarness.name,
|
||||
correctness: summarizeCorrectness(pairs, groups.length),
|
||||
totalTokens: summarizeMetric(pairs, ({ totalTokens }) => totalTokens, groups.length),
|
||||
totalMs: summarizeMetric(pairs, ({ totalMs }) => totalMs, groups.length),
|
||||
estimatedCostUsd: summarizeMetric(pairs, ({ estimatedCostUsd }) => estimatedCostUsd, groups.length),
|
||||
};
|
||||
}
|
||||
|
||||
function compareEvalSet(
|
||||
evalSet: string,
|
||||
data: EvalSetData,
|
||||
): {
|
||||
report: HarnessEvalSetReport;
|
||||
diagnostics: HarnessComparisonDiagnostic[];
|
||||
} {
|
||||
const harnesses = orderedHarnesses(data);
|
||||
const groups = orderedGroups(data);
|
||||
return {
|
||||
report: {
|
||||
evalSet,
|
||||
comparisons: allPairs(harnesses).map(([first, second]) => compareHarnesses(first, second, groups)),
|
||||
},
|
||||
diagnostics: collectDiagnostics(harnesses, groups),
|
||||
};
|
||||
}
|
||||
|
||||
export function summarizeHarnessComparisons(observations: readonly HarnessObservation[]): HarnessComparisonReport {
|
||||
const evalSets: HarnessEvalSetReport[] = [];
|
||||
const diagnostics: HarnessComparisonDiagnostic[] = [];
|
||||
for (const [evalSet, data] of [...groupObservations(observations)].sort(([left], [right]) =>
|
||||
left.localeCompare(right),
|
||||
)) {
|
||||
const comparison = compareEvalSet(evalSet, data);
|
||||
evalSets.push(comparison.report);
|
||||
diagnostics.push(...comparison.diagnostics);
|
||||
}
|
||||
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
evalSets,
|
||||
diagnostics: diagnostics.sort(
|
||||
(left, right) =>
|
||||
left.evalSet.localeCompare(right.evalSet) ||
|
||||
left.file.localeCompare(right.file) ||
|
||||
left.groupKey.localeCompare(right.groupKey) ||
|
||||
left.repetition - right.repetition ||
|
||||
left.seed - right.seed ||
|
||||
left.harness.localeCompare(right.harness),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function formatPercentage(value: number | null): string {
|
||||
return value === null ? "unavailable" : `${(value * 100).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
function formatSigned(value: number, fractionDigits: number): string {
|
||||
return `${value >= 0 ? "+" : ""}${value.toFixed(fractionDigits)}`;
|
||||
}
|
||||
|
||||
function formatMetric(
|
||||
label: string,
|
||||
metric: PairedMetricSummary,
|
||||
formatValue: (value: number) => string,
|
||||
formatDelta: (value: number) => string,
|
||||
): string {
|
||||
if (metric.firstMean === null || metric.secondMean === null || metric.meanDelta === null) {
|
||||
return ` ${label.padEnd(13)}unavailable (${metric.eligiblePairs}/${metric.totalPairs} pairs)`;
|
||||
}
|
||||
return ` ${label.padEnd(13)}${formatValue(metric.secondMean)} vs ${formatValue(metric.firstMean)}, delta ${formatDelta(metric.meanDelta)} (${metric.eligiblePairs}/${metric.totalPairs} pairs)`;
|
||||
}
|
||||
|
||||
export function formatHarnessComparisonReport(report: HarnessComparisonReport): string {
|
||||
if (report.evalSets.every(({ comparisons }) => comparisons.length === 0)) return "";
|
||||
const lines = ["Harness comparisons"];
|
||||
for (const evalSet of report.evalSets) {
|
||||
lines.push(` ${evalSet.evalSet}`);
|
||||
for (const comparison of evalSet.comparisons) {
|
||||
const { correctness } = comparison;
|
||||
const lift = correctness.lift === null ? "unavailable" : `${formatSigned(correctness.lift * 100, 1)} pp`;
|
||||
lines.push(` ${comparison.secondHarness} vs ${comparison.firstHarness}`);
|
||||
lines.push(
|
||||
` ${"Pass rate".padEnd(13)}${formatPercentage(correctness.secondPassRate)} vs ${formatPercentage(correctness.firstPassRate)}, lift ${lift} (${correctness.eligiblePairs}/${correctness.totalPairs} pairs)`,
|
||||
);
|
||||
lines.push(
|
||||
formatMetric(
|
||||
"Tokens",
|
||||
comparison.totalTokens,
|
||||
(value) => value.toFixed(1),
|
||||
(value) => formatSigned(value, 1),
|
||||
),
|
||||
);
|
||||
lines.push(
|
||||
formatMetric(
|
||||
"Latency",
|
||||
comparison.totalMs,
|
||||
(value) => `${value.toFixed(1)}ms`,
|
||||
(value) => `${formatSigned(value, 1)}ms`,
|
||||
),
|
||||
);
|
||||
lines.push(
|
||||
formatMetric(
|
||||
"Est. cost",
|
||||
comparison.estimatedCostUsd,
|
||||
(value) => `$${value.toFixed(4)}`,
|
||||
(value) => `${value >= 0 ? "+" : "-"}$${Math.abs(value).toFixed(4)}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
if (report.diagnostics.length > 0) {
|
||||
lines.push(" Incomplete observations:");
|
||||
for (const diagnostic of report.diagnostics) {
|
||||
lines.push(
|
||||
` ${diagnostic.reason}: ${diagnostic.file}/${diagnostic.testName} repetition ${diagnostic.repetition}, harness ${diagnostic.harness}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
34
packages/evals/test/pi-harness.test.ts
Normal file
34
packages/evals/test/pi-harness.test.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { resolveModelSelection } from "../src/pi-harness.ts";
|
||||
|
||||
describe("resolveModelSelection", () => {
|
||||
it("prefers an explicit harness model over environment defaults", () => {
|
||||
expect(
|
||||
resolveModelSelection(
|
||||
{ provider: "anthropic", id: "claude-opus-4-6" },
|
||||
{ PI_PROVIDER: "openai-codex", PI_MODEL: "gpt-5.6-sol" },
|
||||
),
|
||||
).toEqual({ provider: "anthropic", id: "claude-opus-4-6" });
|
||||
});
|
||||
|
||||
it("uses trimmed environment defaults when the harness has no explicit model", () => {
|
||||
expect(resolveModelSelection(undefined, { PI_PROVIDER: " openai-codex ", PI_MODEL: " gpt-5.6-sol " })).toEqual({
|
||||
provider: "openai-codex",
|
||||
id: "gpt-5.6-sol",
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
[undefined, {}],
|
||||
[undefined, { PI_PROVIDER: "openai-codex" }],
|
||||
[undefined, { PI_MODEL: "gpt-5.6-sol" }],
|
||||
[
|
||||
{ provider: "", id: "gpt-5.6-sol" },
|
||||
{ PI_PROVIDER: "openai-codex", PI_MODEL: "gpt-5.6-sol" },
|
||||
],
|
||||
] as const)("rejects an incomplete model selection", (explicitModel, environment) => {
|
||||
expect(() => resolveModelSelection(explicitModel, environment)).toThrow(
|
||||
"Select a harness model explicitly or set both PI_PROVIDER and PI_MODEL as defaults.",
|
||||
);
|
||||
});
|
||||
});
|
||||
95
packages/evals/test/vitest-evals/artifacts.test.ts
Normal file
95
packages/evals/test/vitest-evals/artifacts.test.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { expect, it } from "vitest";
|
||||
import {
|
||||
getCurrentEvalTest,
|
||||
persistEvalArtifactReferences,
|
||||
recordEvalExtensionSourceArtifact,
|
||||
recordEvalFileArtifact,
|
||||
} from "../../src/vitest-evals/artifacts.ts";
|
||||
|
||||
it("records file and inline Vitest attachments", async ({ task }) => {
|
||||
const root = await mkdtemp(join(tmpdir(), "pi-eval-artifact-test-"));
|
||||
try {
|
||||
const runId = "run-1";
|
||||
const sessionPath = join(root, "session.jsonl");
|
||||
await writeFile(sessionPath, '{"type":"session"}\n', "utf8");
|
||||
|
||||
const test = getCurrentEvalTest();
|
||||
await recordEvalFileArtifact(test, runId, { name: "session", path: sessionPath });
|
||||
await recordEvalExtensionSourceArtifact(task, runId, "export default function () {}\n");
|
||||
expect(test.artifacts).toContainEqual(
|
||||
expect.objectContaining({
|
||||
type: "@earendil-works/pi-evals:file",
|
||||
runId,
|
||||
attachments: [
|
||||
expect.objectContaining({
|
||||
name: "session",
|
||||
path: expect.any(String),
|
||||
}),
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(test.artifacts).toContainEqual(
|
||||
expect.objectContaining({
|
||||
type: "@earendil-works/pi-evals:extension-source",
|
||||
runId,
|
||||
attachments: [
|
||||
expect.objectContaining({
|
||||
name: "hello.ts",
|
||||
body: "export default function () {}\n",
|
||||
bodyEncoding: "utf-8",
|
||||
contentType: "text/typescript",
|
||||
}),
|
||||
],
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("persists and selects attachments belonging to the reported run", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "pi-eval-artifact-report-test-"));
|
||||
try {
|
||||
const references = await persistEvalArtifactReferences(
|
||||
[
|
||||
{
|
||||
type: "@earendil-works/pi-evals:file",
|
||||
runId: "run-1",
|
||||
attachments: [{ name: "session", path: join(root, "sessions", "run-1.jsonl") }],
|
||||
},
|
||||
{
|
||||
type: "@earendil-works/pi-evals:file",
|
||||
runId: "run-2",
|
||||
attachments: [{ name: "session", path: "/eval/sessions/run-2.jsonl" }],
|
||||
},
|
||||
{
|
||||
type: "@earendil-works/pi-evals:extension-source",
|
||||
runId: "run-1",
|
||||
attachments: [
|
||||
{
|
||||
name: "hello.ts",
|
||||
body: "export default function () {}\n",
|
||||
bodyEncoding: "utf-8",
|
||||
contentType: "text/typescript",
|
||||
},
|
||||
],
|
||||
},
|
||||
{ type: "internal:annotation", annotation: { message: "other", type: "info" } },
|
||||
],
|
||||
"run-1",
|
||||
root,
|
||||
);
|
||||
expect(references).toEqual([
|
||||
{ name: "session", path: "sessions/run-1.jsonl" },
|
||||
{ name: "hello.ts", path: expect.stringMatching(/^sources\/[a-f0-9]{64}\/hello\.ts$/) },
|
||||
]);
|
||||
const source = references.find(({ name }) => name === "hello.ts");
|
||||
if (!source) throw new TypeError("Expected a persisted extension source reference.");
|
||||
expect(await readFile(join(root, source.path), "utf8")).toBe("export default function () {}\n");
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
97
packages/evals/test/vitest-evals/harness-table.test.ts
Normal file
97
packages/evals/test/vitest-evals/harness-table.test.ts
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
import { afterAll, describe, expect, it } from "vitest";
|
||||
import { createHarness, describeEval } from "vitest-evals";
|
||||
import {
|
||||
deriveEvalGroupKey,
|
||||
EVAL_HARNESS_ITERATION_ARTIFACT,
|
||||
evalHarnessTable,
|
||||
parseEvalHarnessIterationArtifact,
|
||||
} from "../../src/vitest-evals/harness-table.ts";
|
||||
|
||||
describe("deriveEvalGroupKey", () => {
|
||||
it("combines a trimmed string input ID with repetition and seed", () => {
|
||||
expect(deriveEvalGroupKey({ id: " input-1 ", prompt: "hello" }, 2, 42)).toBe(JSON.stringify(["input-1", 2, 42]));
|
||||
});
|
||||
|
||||
it("hashes canonical JSON independently of object key order", () => {
|
||||
expect(deriveEvalGroupKey({ first: 1, second: [true, "value"] }, 1, 42)).toBe(
|
||||
deriveEvalGroupKey({ second: [true, "value"], first: 1 }, 1, 42),
|
||||
);
|
||||
expect(deriveEvalGroupKey({ first: 1 }, 1, 42)).not.toBe(deriveEvalGroupKey({ first: 2 }, 1, 42));
|
||||
expect(deriveEvalGroupKey({ first: 1 }, 1, 42)).not.toBe(deriveEvalGroupKey({ first: 1 }, 2, 42));
|
||||
});
|
||||
|
||||
it("rejects non-JSON and circular input", () => {
|
||||
const circular: { self?: unknown } = {};
|
||||
circular.self = circular;
|
||||
expect(() => deriveEvalGroupKey(new Date(0), 1, 42)).toThrow("only plain objects and arrays");
|
||||
expect(() => deriveEvalGroupKey(Array(1), 1, 42)).toThrow("must be JSON-serializable");
|
||||
expect(() => deriveEvalGroupKey(circular, 1, 42)).toThrow("must not contain circular references");
|
||||
});
|
||||
});
|
||||
|
||||
const observations: Array<{
|
||||
harness: string;
|
||||
inputId: string;
|
||||
repetition: number;
|
||||
plannedOrder: number;
|
||||
}> = [];
|
||||
|
||||
function createFakeHarness(name: string) {
|
||||
return createHarness<{ id: string }, { harness: string; inputId: string }>({
|
||||
name,
|
||||
run: ({ input }) => ({
|
||||
output: { harness: name, inputId: input.id },
|
||||
events: [
|
||||
{ type: "message", role: "user", content: input.id },
|
||||
{ type: "message", role: "assistant", content: name },
|
||||
],
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
const harnessTable = evalHarnessTable(
|
||||
"local multi-harness eval",
|
||||
[createFakeHarness("withoutSkill"), createFakeHarness("withSkill")],
|
||||
{ repetitions: 2, seed: 42 },
|
||||
);
|
||||
|
||||
describe.for(harnessTable)("$name repetition $repetition", ({ harness, repetition: plannedRepetition }) => {
|
||||
describeEval("local multi-harness eval", { harness }, (it) => {
|
||||
it.for([{ id: "first" }, { id: "second" }])("$id", async ({ id }, { run }) => {
|
||||
const result = await run({ id });
|
||||
const runMetadata = parseEvalHarnessIterationArtifact(result.artifacts?.[EVAL_HARNESS_ITERATION_ARTIFACT]);
|
||||
|
||||
expect(result.output.inputId).toBe(id);
|
||||
expect(runMetadata).toEqual({
|
||||
schemaVersion: 1,
|
||||
evalSet: "local multi-harness eval",
|
||||
groupKey: deriveEvalGroupKey({ id }, plannedRepetition, 42),
|
||||
harness: result.output.harness,
|
||||
harnesses: ["withoutSkill", "withSkill"],
|
||||
repetition: plannedRepetition,
|
||||
seed: 42,
|
||||
plannedOrder: expect.any(Number),
|
||||
});
|
||||
if (!runMetadata) throw new TypeError("Expected typed harness-iteration metadata.");
|
||||
const { harness: harnessName, repetition, plannedOrder } = runMetadata;
|
||||
observations.push({ harness: harnessName, inputId: id, repetition, plannedOrder });
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
expect(observations).toHaveLength(8);
|
||||
for (const inputId of ["first", "second"]) {
|
||||
for (const repetition of [1, 2]) {
|
||||
expect(
|
||||
observations
|
||||
.filter((observation) => observation.inputId === inputId && observation.repetition === repetition)
|
||||
.map((observation) => observation.harness)
|
||||
.sort(),
|
||||
).toEqual(["withSkill", "withoutSkill"]);
|
||||
}
|
||||
}
|
||||
expect(
|
||||
[...new Set(observations.map((observation) => observation.plannedOrder))].sort((left, right) => left - right),
|
||||
).toEqual([1, 2, 3, 4]);
|
||||
});
|
||||
230
packages/evals/test/vitest-evals/summary.test.ts
Normal file
230
packages/evals/test/vitest-evals/summary.test.ts
Normal file
|
|
@ -0,0 +1,230 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
formatHarnessComparisonReport,
|
||||
type HarnessObservation,
|
||||
type HarnessObservationOutcome,
|
||||
summarizeHarnessComparisons,
|
||||
} from "../../src/vitest-evals/summary.ts";
|
||||
|
||||
type ObservationResult = "passed" | "failed" | Exclude<HarnessObservationOutcome, "scored">;
|
||||
|
||||
function observation(
|
||||
harness: string,
|
||||
testName: string,
|
||||
result: ObservationResult,
|
||||
metrics: Pick<HarnessObservation, "totalTokens" | "totalMs" | "estimatedCostUsd"> = {},
|
||||
harnesses: string[] = ["without-tools", "with-tools"],
|
||||
): HarnessObservation {
|
||||
const base = {
|
||||
evalSet: "tool access",
|
||||
groupKey: JSON.stringify([testName, 1, 42]),
|
||||
testName,
|
||||
file: "src/tool-access.eval.ts",
|
||||
harness,
|
||||
harnesses,
|
||||
repetition: 1,
|
||||
seed: 42,
|
||||
...metrics,
|
||||
};
|
||||
if (result === "passed" || result === "failed") {
|
||||
return { ...base, outcome: "scored", score: result === "passed" ? 1 : 0 };
|
||||
}
|
||||
return { ...base, outcome: result };
|
||||
}
|
||||
|
||||
describe("summarizeHarnessComparisons", () => {
|
||||
it("computes paired correctness lift separately from efficiency deltas", () => {
|
||||
const report = summarizeHarnessComparisons([
|
||||
observation("without-tools", "create", "failed", {
|
||||
totalTokens: 100,
|
||||
totalMs: 1000,
|
||||
estimatedCostUsd: 0.01,
|
||||
}),
|
||||
observation("with-tools", "create", "passed", {
|
||||
totalTokens: 120,
|
||||
totalMs: 800,
|
||||
estimatedCostUsd: 0.02,
|
||||
}),
|
||||
observation("without-tools", "inspect", "passed", { totalTokens: 200 }),
|
||||
observation("with-tools", "inspect", "passed", { totalTokens: 180 }),
|
||||
]);
|
||||
|
||||
expect(report.evalSets).toHaveLength(1);
|
||||
expect(report.evalSets[0]?.comparisons).toEqual([
|
||||
expect.objectContaining({
|
||||
firstHarness: "without-tools",
|
||||
secondHarness: "with-tools",
|
||||
correctness: {
|
||||
totalPairs: 2,
|
||||
eligiblePairs: 2,
|
||||
firstPassRate: 0.5,
|
||||
secondPassRate: 1,
|
||||
lift: 0.5,
|
||||
firstWins: 0,
|
||||
secondWins: 1,
|
||||
ties: 1,
|
||||
},
|
||||
totalTokens: {
|
||||
totalPairs: 2,
|
||||
eligiblePairs: 2,
|
||||
firstMean: 150,
|
||||
secondMean: 150,
|
||||
meanDelta: 0,
|
||||
},
|
||||
totalMs: {
|
||||
totalPairs: 2,
|
||||
eligiblePairs: 1,
|
||||
firstMean: 1000,
|
||||
secondMean: 800,
|
||||
meanDelta: -200,
|
||||
},
|
||||
estimatedCostUsd: {
|
||||
totalPairs: 2,
|
||||
eligiblePairs: 1,
|
||||
firstMean: 0.01,
|
||||
secondMean: 0.02,
|
||||
meanDelta: 0.01,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
expect(report.diagnostics).toEqual([]);
|
||||
});
|
||||
|
||||
it("reports missing observations without coercing them to failures or zero telemetry", () => {
|
||||
const report = summarizeHarnessComparisons([
|
||||
observation("without-tools", "create", "failed"),
|
||||
observation("with-tools", "create", "passed"),
|
||||
observation("without-tools", "inspect", "passed"),
|
||||
]);
|
||||
const comparison = report.evalSets[0]?.comparisons[0];
|
||||
|
||||
expect(comparison?.correctness).toEqual({
|
||||
totalPairs: 2,
|
||||
eligiblePairs: 1,
|
||||
firstPassRate: 0,
|
||||
secondPassRate: 1,
|
||||
lift: 1,
|
||||
firstWins: 0,
|
||||
secondWins: 1,
|
||||
ties: 0,
|
||||
});
|
||||
expect(comparison?.totalTokens).toEqual({
|
||||
totalPairs: 2,
|
||||
eligiblePairs: 0,
|
||||
firstMean: null,
|
||||
secondMean: null,
|
||||
meanDelta: null,
|
||||
});
|
||||
expect(report.diagnostics).toContainEqual(
|
||||
expect.objectContaining({
|
||||
testName: "inspect",
|
||||
harness: "with-tools",
|
||||
reason: "missing-observation",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps identical inputs in different test files separate", () => {
|
||||
const report = summarizeHarnessComparisons([
|
||||
observation("without-tools", "shared", "failed"),
|
||||
observation("with-tools", "shared", "passed"),
|
||||
{ ...observation("without-tools", "shared", "passed"), file: "src/other.eval.ts" },
|
||||
{ ...observation("with-tools", "shared", "passed"), file: "src/other.eval.ts" },
|
||||
]);
|
||||
|
||||
expect(report.evalSets[0]?.comparisons[0]?.correctness).toEqual(
|
||||
expect.objectContaining({ totalPairs: 2, eligiblePairs: 2 }),
|
||||
);
|
||||
expect(report.diagnostics).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not score harness errors as correctness failures", () => {
|
||||
const report = summarizeHarnessComparisons([
|
||||
observation("without-tools", "create", "errored", { totalTokens: 100 }),
|
||||
observation("with-tools", "create", "passed", { totalTokens: 100 }),
|
||||
]);
|
||||
|
||||
expect(report.evalSets[0]?.comparisons[0]?.correctness).toEqual(
|
||||
expect.objectContaining({ totalPairs: 1, eligiblePairs: 0 }),
|
||||
);
|
||||
expect(report.evalSets[0]?.comparisons[0]?.totalTokens.eligiblePairs).toBe(0);
|
||||
expect(report.diagnostics).toContainEqual(
|
||||
expect.objectContaining({ harness: "without-tools", reason: "harness-error" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not derive correctness from completed Vitest tests without judge scores", () => {
|
||||
const withoutScore = observation("without-tools", "create", "unscored");
|
||||
const withScore = observation("with-tools", "create", "unscored");
|
||||
|
||||
const report = summarizeHarnessComparisons([withoutScore, withScore]);
|
||||
|
||||
expect(report.evalSets[0]?.comparisons[0]?.correctness.eligiblePairs).toBe(0);
|
||||
expect(report.diagnostics).toEqual([
|
||||
expect.objectContaining({ harness: "with-tools", reason: "missing-score" }),
|
||||
expect.objectContaining({ harness: "without-tools", reason: "missing-score" }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("creates every pair in declared harness order", () => {
|
||||
const harnesses = ["first", "second", "third"];
|
||||
const report = summarizeHarnessComparisons([
|
||||
observation("first", "input", "passed", {}, harnesses),
|
||||
observation("second", "input", "passed", {}, harnesses),
|
||||
observation("third", "input", "passed", {}, harnesses),
|
||||
]);
|
||||
|
||||
expect(
|
||||
report.evalSets[0]?.comparisons.map(({ firstHarness, secondHarness }) => [firstHarness, secondHarness]),
|
||||
).toEqual([
|
||||
["first", "second"],
|
||||
["first", "third"],
|
||||
["second", "third"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("retains a declared harness with no completed observations", () => {
|
||||
const report = summarizeHarnessComparisons([observation("without-tools", "create", "failed")]);
|
||||
|
||||
expect(report.evalSets[0]?.comparisons).toHaveLength(1);
|
||||
expect(report.evalSets[0]?.comparisons[0]?.correctness.eligiblePairs).toBe(0);
|
||||
expect(report.diagnostics).toContainEqual(
|
||||
expect.objectContaining({
|
||||
testName: "create",
|
||||
harness: "with-tools",
|
||||
reason: "missing-observation",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("reports duplicate and unscorable observations once across multiple harness pairs", () => {
|
||||
const harnesses = ["first", "second", "third"];
|
||||
const report = summarizeHarnessComparisons([
|
||||
observation("first", "duplicate", "passed", {}, harnesses),
|
||||
observation("first", "duplicate", "failed", {}, harnesses),
|
||||
observation("second", "duplicate", "passed", {}, harnesses),
|
||||
observation("third", "duplicate", "passed", {}, harnesses),
|
||||
observation("first", "skipped", "skipped", {}, harnesses),
|
||||
observation("second", "skipped", "passed", {}, harnesses),
|
||||
observation("third", "skipped", "passed", {}, harnesses),
|
||||
]);
|
||||
|
||||
expect(report.diagnostics.filter(({ reason }) => reason === "duplicate-observation")).toEqual([
|
||||
expect.objectContaining({ testName: "duplicate", harness: "first" }),
|
||||
]);
|
||||
expect(report.diagnostics.filter(({ reason }) => reason === "unscorable-outcome")).toEqual([
|
||||
expect.objectContaining({ testName: "skipped", harness: "first" }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("formats lift and telemetry availability for the terminal report", () => {
|
||||
const report = summarizeHarnessComparisons([
|
||||
observation("without-tools", "create", "failed"),
|
||||
observation("with-tools", "create", "passed"),
|
||||
]);
|
||||
|
||||
expect(formatHarnessComparisonReport(report)).toContain("with-tools vs without-tools");
|
||||
expect(formatHarnessComparisonReport(report)).toContain("lift +100.0 pp");
|
||||
expect(formatHarnessComparisonReport(report)).toContain("Tokens unavailable (0/1 pairs)");
|
||||
});
|
||||
});
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import { join } from "node:path";
|
||||
import { defineConfig, mergeConfig } from "vitest/config";
|
||||
import baseConfig, { workspaceSourcePaths } from "../../vitest.base.ts";
|
||||
|
||||
|
|
@ -9,8 +10,9 @@ export default mergeConfig(
|
|||
fileParallelism: false,
|
||||
include: ["src/**/*.eval.ts"],
|
||||
testTimeout: 120000,
|
||||
attachmentsDir: join(process.env.PI_EVAL_ARTIFACT_DIR ?? ".eval", "sessions"),
|
||||
hookTimeout: 30000,
|
||||
reporters: ["vitest-evals/reporter"],
|
||||
reporters: ["vitest-evals/reporter", "./src/vitest-evals/reporter.ts"],
|
||||
},
|
||||
resolve: {
|
||||
alias: [{ find: /^@earendil-works\/pi-coding-agent$/, replacement: workspaceSourcePaths.codingAgentIndex }],
|
||||
|
|
|
|||
15
packages/evals/vitest.test.config.ts
Normal file
15
packages/evals/vitest.test.config.ts
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
import { defineConfig, mergeConfig } from "vitest/config";
|
||||
import baseConfig, { workspaceSourcePaths } from "../../vitest.base.ts";
|
||||
|
||||
export default mergeConfig(
|
||||
baseConfig,
|
||||
defineConfig({
|
||||
test: {
|
||||
include: ["test/**/*.test.ts"],
|
||||
attachmentsDir: ".eval/test-attachments",
|
||||
},
|
||||
resolve: {
|
||||
alias: [{ find: /^@earendil-works\/pi-coding-agent$/, replacement: workspaceSourcePaths.codingAgentIndex }],
|
||||
},
|
||||
}),
|
||||
);
|
||||
Loading…
Add table
Add a link
Reference in a new issue