diff --git a/packages/evals/README.md b/packages/evals/README.md index 8d3ce970a..c18c011af 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -120,8 +120,6 @@ const harnessTable = evalHarnessTable( { baseline: withoutTargetSkillHarness, candidate: withTargetSkillHarness, - }, - { repetitions: 6, seed: 42, }, diff --git a/packages/evals/src/extensions.eval.ts b/packages/evals/src/extensions.eval.ts index d93dfafa4..e58e7575a 100644 --- a/packages/evals/src/extensions.eval.ts +++ b/packages/evals/src/extensions.eval.ts @@ -97,14 +97,11 @@ const ExtensionAuthoringJudge = createJudge { describeEval( diff --git a/packages/evals/src/vitest-evals/harness-table.ts b/packages/evals/src/vitest-evals/harness-table.ts index 8824b8ccd..694fd3bec 100644 --- a/packages/evals/src/vitest-evals/harness-table.ts +++ b/packages/evals/src/vitest-evals/harness-table.ts @@ -29,24 +29,23 @@ export type EvalHarnessTableRow = plannedOrder: number; }; -export type EvalHarnessTableOptions = { +export type EvalHarnessTablePairOptions = { + baseline: Harness; + candidate: Harness; repetitions?: number; seed?: number; }; -export type EvalHarnessTablePair = { - baseline: Harness; - candidate: Harness; -}; - -export type EvalHarnessTableCandidates = { +export type EvalHarnessTableCandidatesOptions = { baseline: Harness; candidates: readonly Harness[]; + repetitions?: number; + seed?: number; }; -type EvalHarnessTableComparison = - | EvalHarnessTablePair - | EvalHarnessTableCandidates; +export type EvalHarnessTableOptions = + | EvalHarnessTablePairOptions + | EvalHarnessTableCandidatesOptions; type EvalHarnessIterationPlan = Omit; @@ -180,27 +179,24 @@ function withIterationArtifact( export function evalHarnessTable( evalSet: string, - comparison: EvalHarnessTablePair, - options: EvalHarnessTableOptions, + options: EvalHarnessTablePairOptions, ): EvalHarnessTableRow[]; export function evalHarnessTable( evalSet: string, - comparison: EvalHarnessTableCandidates, - options: EvalHarnessTableOptions, + options: EvalHarnessTableCandidatesOptions, ): EvalHarnessTableRow[]; export function evalHarnessTable( evalSet: string, - comparison: EvalHarnessTableComparison, - options: EvalHarnessTableOptions, + options: EvalHarnessTableOptions, ): EvalHarnessTableRow[] { const repetitions = options.repetitions ?? 1; const seed = options.seed ?? randomBytes(4).readUInt32LE(); - const candidates = "candidate" in comparison ? [comparison.candidate] : comparison.candidates; - validateOptions(evalSet, comparison.baseline, candidates, repetitions, seed); + const candidates = "candidate" in options ? [options.candidate] : options.candidates; + validateOptions(evalSet, options.baseline, candidates, repetitions, seed); const random = createSeededRandom(seed); const rows: EvalHarnessTableRow[] = []; - const harnesses = [comparison.baseline, ...candidates]; + const harnesses = [options.baseline, ...candidates]; let plannedOrder = 1; for (let repetition = 1; repetition <= repetitions; repetition += 1) { const randomizedHarnesses = [...harnesses]; @@ -216,7 +212,7 @@ export function evalHarnessTable( schemaVersion: 1, evalSet, harness: harness.name, - baseline: comparison.baseline.name, + baseline: options.baseline.name, candidates: candidates.map(({ name }) => name), repetition, seed, diff --git a/packages/evals/src/vitest-evals/summary.ts b/packages/evals/src/vitest-evals/summary.ts index 99af02839..7d0c64d06 100644 --- a/packages/evals/src/vitest-evals/summary.ts +++ b/packages/evals/src/vitest-evals/summary.ts @@ -370,8 +370,11 @@ function formatMetric( if (metric.baselineMean === null || metric.candidateMean === null || metric.meanDelta === null) { return formatReportLine(label, `${styleText("yellow", "unavailable")}${coverage}`); } - const delta = colorDelta(metric.meanDelta, `delta ${formatDelta(metric.meanDelta)}`, false); - const values = styleText("gray", `(${formatValue(metric.candidateMean)}, ${formatValue(metric.baselineMean)})`); + const delta = colorDelta(metric.meanDelta, formatDelta(metric.meanDelta), false); + const values = styleText( + "gray", + `(candidate ${formatValue(metric.candidateMean)}, baseline ${formatValue(metric.baselineMean)})`, + ); return formatReportLine(label, `${delta} ${values}${coverage}`); } @@ -394,10 +397,10 @@ export function formatHarnessComparisonReport(report: HarnessComparisonReport): lines.push(formatReportLine("Pass rate", styleText("yellow", "unavailable"))); } else { const lift = correctness.lift * 100; - const delta = colorDelta(lift, `delta ${formatSigned(lift, 1)} pp`, true); + const delta = colorDelta(lift, `${formatSigned(lift, 1)} pp`, true); const values = styleText( "gray", - `(${formatPercentage(correctness.candidatePassRate)}, ${formatPercentage(correctness.baselinePassRate)})`, + `(candidate ${formatPercentage(correctness.candidatePassRate)}, baseline ${formatPercentage(correctness.baselinePassRate)})`, ); lines.push(formatReportLine("Pass rate", `${delta} ${values}`)); } diff --git a/packages/evals/test/vitest-evals/harness-table.test.ts b/packages/evals/test/vitest-evals/harness-table.test.ts index db77f9d40..4d0502b33 100644 --- a/packages/evals/test/vitest-evals/harness-table.test.ts +++ b/packages/evals/test/vitest-evals/harness-table.test.ts @@ -43,14 +43,12 @@ function createFakeHarness(name: string) { }); } -const harnessTable = evalHarnessTable( - "local multi-harness eval", - { - baseline: createFakeHarness("withoutSkill"), - candidates: [createFakeHarness("withSkill")], - }, - { repetitions: 2, seed: 42 }, -); +const harnessTable = evalHarnessTable("local multi-harness eval", { + baseline: createFakeHarness("withoutSkill"), + candidates: [createFakeHarness("withSkill")], + repetitions: 2, + seed: 42, +}); describe("evalHarnessTable", () => { it("plans seeded repetitions in deterministic order", () => { @@ -65,11 +63,11 @@ describe("evalHarnessTable", () => { }); it("accepts a singular candidate", () => { - const rows = evalHarnessTable( - "singular candidate", - { baseline: createFakeHarness("baseline"), candidate: createFakeHarness("candidate") }, - { seed: 42 }, - ); + const rows = evalHarnessTable("singular candidate", { + baseline: createFakeHarness("baseline"), + candidate: createFakeHarness("candidate"), + seed: 42, + }); expect(rows.map(({ name }) => name)).toEqual(["baseline", "candidate"]); }); diff --git a/packages/evals/test/vitest-evals/summary.test.ts b/packages/evals/test/vitest-evals/summary.test.ts index 077a1d2b1..68ed1f4e8 100644 --- a/packages/evals/test/vitest-evals/summary.test.ts +++ b/packages/evals/test/vitest-evals/summary.test.ts @@ -226,8 +226,8 @@ describe("summarizeHarnessComparisons", () => { expect(formatted).toContain("Eval Comparisons"); expect(formatted).toContain(" Baseline without-tools"); expect(formatted).toContain("Candidate with-tools (1/1 pairs)"); - expect(formatted).toContain("Pass rate delta +100.0 pp (100.0%, 0.0%)"); + expect(formatted).toContain("Pass rate +100.0 pp (candidate 100.0%, baseline 0.0%)"); expect(formatted).toContain(" Tokens unavailable"); - expect(formatted).toContain(" Latency delta -4159.5ms (30694.2ms, 34853.7ms)"); + expect(formatted).toContain(" Latency -4159.5ms (candidate 30694.2ms, baseline 34853.7ms)"); }); });