From fce7445b1c14451cb3b7d201bff2df7dd7f91b3e Mon Sep 17 00:00:00 2001 From: Christian Klotz Date: Thu, 30 Jul 2026 10:58:07 +0300 Subject: [PATCH] fix(coding-agent): make eval comparison direction explicit --- packages/evals/README.md | 17 +- packages/evals/src/extensions.eval.ts | 8 +- .../evals/src/vitest-evals/harness-table.ts | 51 ++++- packages/evals/src/vitest-evals/reporter.ts | 5 +- packages/evals/src/vitest-evals/summary.ts | 175 ++++++++++-------- .../test/vitest-evals/harness-table.test.ts | 18 +- .../evals/test/vitest-evals/summary.test.ts | 76 ++++---- 7 files changed, 205 insertions(+), 145 deletions(-) diff --git a/packages/evals/README.md b/packages/evals/README.md index d38978f6a..8d3ce970a 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -117,7 +117,10 @@ const TargetTaskJudge = createJudge("TargetTaskJudge", ({ output const harnessTable = evalHarnessTable( "target skill effectiveness", - [withoutTargetSkillHarness, withTargetSkillHarness], + { + baseline: withoutTargetSkillHarness, + candidate: withTargetSkillHarness, + }, { repetitions: 6, seed: 42, @@ -142,12 +145,12 @@ registers that snapshot against the explicit Vitest test task before reporters r Harness names must be stable and unique within an eval set. The grouping key combines repetition and seed with a non-empty string `input.id` when available, otherwise with a SHA-256 hash of strict canonical JSON input. Execution order -is seeded and randomized, while harness declaration order determines comparison direction. For each matched input and -repetition, the reporter computes pass-rate lift from each run's recorded average judge score, treating a score of at -least `1` as passing. The lift is the later-declared harness's pass rate minus the earlier-declared harness's pass rate, -in percentage points. Missing judge scores are reported as incomplete observations. Tokens, latency, and estimated cost -remain separate paired deltas; missing telemetry remains unavailable. Sets -with more than two harnesses report every pair. +is seeded and randomized independently of comparison direction. Use `candidate` for one treatment or `candidates` for +multiple treatments. Each candidate is compared only with the declared baseline. For each matched input and repetition, +the reporter computes pass-rate lift from each run's recorded average judge score, treating a score of at least `1` as +passing. Lift is the candidate pass rate minus the baseline pass rate, in percentage points. Missing judge scores are +reported as incomplete observations. Tokens, latency, and estimated cost remain separate candidate-minus-baseline paired +deltas; missing telemetry remains unavailable. See the [`skill-eval-harness`](https://github.com/adewale/skill-eval-harness/) guidance for comparative-eval methodology, repetition strategy, trustworthy judges, and telemetry interpretation. diff --git a/packages/evals/src/extensions.eval.ts b/packages/evals/src/extensions.eval.ts index 1e0a0caaf..d93dfafa4 100644 --- a/packages/evals/src/extensions.eval.ts +++ b/packages/evals/src/extensions.eval.ts @@ -99,10 +99,10 @@ const ExtensionAuthoringJudge = createJudge = { + baseline: Harness; + candidate: Harness; +}; + +export type EvalHarnessTableCandidates = { + baseline: Harness; + candidates: readonly Harness[]; +}; + +type EvalHarnessTableComparison = + | EvalHarnessTablePair + | EvalHarnessTableCandidates; + type EvalHarnessIterationPlan = Omit; export function parseEvalHarnessIterationArtifact( value: JsonValue | undefined, ): EvalHarnessIterationArtifact | undefined { if (value === null || value === undefined || typeof value !== "object" || Array.isArray(value)) return undefined; - const { schemaVersion, evalSet, groupKey, harness, harnesses, repetition, seed, plannedOrder } = value; + const { schemaVersion, evalSet, groupKey, harness, baseline, candidates, repetition, seed, plannedOrder } = value; if ( schemaVersion !== 1 || typeof evalSet !== "string" || typeof groupKey !== "string" || typeof harness !== "string" || - !Array.isArray(harnesses) || - !harnesses.every((name): name is string => typeof name === "string") || + typeof baseline !== "string" || + !Array.isArray(candidates) || + !candidates.every((name): name is string => typeof name === "string") || typeof repetition !== "number" || typeof seed !== "number" || typeof plannedOrder !== "number" ) { return undefined; } - return { schemaVersion, evalSet, groupKey, harness, harnesses, repetition, seed, plannedOrder }; + return { schemaVersion, evalSet, groupKey, harness, baseline, candidates, repetition, seed, plannedOrder }; } function createSeededRandom(seed: number): () => number { @@ -117,12 +133,14 @@ export function deriveEvalGroupKey(input: unknown, repetition: number, seed: num function validateOptions( evalSet: string, - harnesses: readonly Harness[], + baseline: Harness, + candidates: readonly Harness[], repetitions: number, seed: number, ): void { if (!evalSet.trim()) throw new TypeError("evalSet must not be empty."); - if (harnesses.length < 2) throw new TypeError("At least two harnesses are required."); + if (candidates.length === 0) throw new TypeError("At least one candidate harness is required."); + const harnesses = [baseline, ...candidates]; const names = new Set(harnesses.map((harness) => harness.name)); if (names.size !== harnesses.length) throw new TypeError("Harness names must be unique within an eval set."); if (!Number.isSafeInteger(repetitions) || repetitions < 1) { @@ -162,15 +180,27 @@ function withIterationArtifact( export function evalHarnessTable( evalSet: string, - harnesses: readonly Harness[], + comparison: EvalHarnessTablePair, + options: EvalHarnessTableOptions, +): EvalHarnessTableRow[]; +export function evalHarnessTable( + evalSet: string, + comparison: EvalHarnessTableCandidates, + options: EvalHarnessTableOptions, +): EvalHarnessTableRow[]; +export function evalHarnessTable( + evalSet: string, + comparison: EvalHarnessTableComparison, options: EvalHarnessTableOptions, ): EvalHarnessTableRow[] { const repetitions = options.repetitions ?? 1; const seed = options.seed ?? randomBytes(4).readUInt32LE(); - validateOptions(evalSet, harnesses, repetitions, seed); + const candidates = "candidate" in comparison ? [comparison.candidate] : comparison.candidates; + validateOptions(evalSet, comparison.baseline, candidates, repetitions, seed); const random = createSeededRandom(seed); const rows: EvalHarnessTableRow[] = []; + const harnesses = [comparison.baseline, ...candidates]; let plannedOrder = 1; for (let repetition = 1; repetition <= repetitions; repetition += 1) { const randomizedHarnesses = [...harnesses]; @@ -186,7 +216,8 @@ export function evalHarnessTable( schemaVersion: 1, evalSet, harness: harness.name, - harnesses: harnesses.map(({ name }) => name), + baseline: comparison.baseline.name, + candidates: candidates.map(({ name }) => name), repetition, seed, plannedOrder, diff --git a/packages/evals/src/vitest-evals/reporter.ts b/packages/evals/src/vitest-evals/reporter.ts index 17bce8fca..7aa45b796 100644 --- a/packages/evals/src/vitest-evals/reporter.ts +++ b/packages/evals/src/vitest-evals/reporter.ts @@ -65,7 +65,8 @@ function collectHarnessObservations(modules: ReadonlyArray): Harness testName: test.name, file: module.relativeModuleId, harness: iteration.harness, - harnesses: iteration.harnesses, + baseline: iteration.baseline, + candidates: iteration.candidates, repetition: iteration.repetition, seed: iteration.seed, ...(run.usage.totalTokens === undefined ? {} : { totalTokens: run.usage.totalTokens }), @@ -101,7 +102,7 @@ export default class EvalHarnessReporter implements Reporter { reason: TestRunEndReason, ): void { if (reason === "interrupted") { - this.vitest?.logger.log("\nHarness comparisons unavailable: test run interrupted."); + this.vitest?.logger.log("\nEval comparisons unavailable: test run interrupted."); return; } const report = summarizeHarnessComparisons(collectHarnessObservations(modules)); diff --git a/packages/evals/src/vitest-evals/summary.ts b/packages/evals/src/vitest-evals/summary.ts index 76f15ef0b..99af02839 100644 --- a/packages/evals/src/vitest-evals/summary.ts +++ b/packages/evals/src/vitest-evals/summary.ts @@ -8,7 +8,8 @@ type HarnessObservationBase = { testName: string; file: string; harness: string; - harnesses: string[]; + baseline: string; + candidates: string[]; repetition: number; seed: number; totalTokens?: number; @@ -22,25 +23,25 @@ export type HarnessObservation = HarnessObservationBase & export type PairedMetricSummary = { totalPairs: number; eligiblePairs: number; - firstMean: number | null; - secondMean: number | null; + baselineMean: number | null; + candidateMean: number | null; meanDelta: number | null; }; export type CorrectnessLiftSummary = { totalPairs: number; eligiblePairs: number; - firstPassRate: number | null; - secondPassRate: number | null; + baselinePassRate: number | null; + candidatePassRate: number | null; lift: number | null; - firstWins: number; - secondWins: number; + baselineWins: number; + candidateWins: number; ties: number; }; export type HarnessPairComparison = { - firstHarness: string; - secondHarness: string; + baseline: string; + candidate: string; correctness: CorrectnessLiftSummary; totalTokens: PairedMetricSummary; totalMs: PairedMetricSummary; @@ -85,13 +86,14 @@ type ObservationGroup = { }; type EvalSetData = { - harnessesByName: Map; + baseline: HarnessDescriptor; + candidatesByName: Map; groupsByKey: Map; }; type ObservationPair = { - first: HarnessObservation; - second: HarnessObservation; + baseline: HarnessObservation; + candidate: HarnessObservation; }; function getOrCreate(map: Map, key: K, create: () => V): V { @@ -110,27 +112,18 @@ function preciseDifference(left: number, right: number): number { return Number((left - right).toPrecision(15)); } -function allPairs(values: readonly T[]): Array<[T, T]> { - const pairs: Array<[T, T]> = []; - for (let first = 0; first < values.length; first += 1) { - for (let second = first + 1; second < values.length; second += 1) { - pairs.push([values[first], values[second]]); - } - } - return pairs; -} - function groupObservations(observations: readonly HarnessObservation[]): Map { const evalSets = new Map(); for (const observation of observations) { const evalSet = getOrCreate(evalSets, observation.evalSet, () => ({ - harnessesByName: new Map(), + baseline: { name: observation.baseline, index: 0 }, + candidatesByName: new Map(), groupsByKey: new Map(), })); - for (const [index, name] of observation.harnesses.entries()) { - const existing = evalSet.harnessesByName.get(name); - if (!existing || index < existing.index) evalSet.harnessesByName.set(name, { name, index }); + for (const [index, name] of observation.candidates.entries()) { + const existing = evalSet.candidatesByName.get(name); + if (!existing || index < existing.index) evalSet.candidatesByName.set(name, { name, index }); } const group = getOrCreate( @@ -152,7 +145,16 @@ function groupObservations(observations: readonly HarnessObservation[]): Map left.index - right.index || left.name.localeCompare(right.name), + ), + ]; +} + +function orderedCandidates(evalSet: EvalSetData): HarnessDescriptor[] { + return [...evalSet.candidatesByName.values()].sort( (left, right) => left.index - right.index || left.name.localeCompare(right.name), ); } @@ -199,14 +201,16 @@ function collectDiagnostics( function pairObservations( groups: readonly ObservationGroup[], - firstHarness: string, - secondHarness: string, + baselineHarness: string, + candidateHarness: string, ): ObservationPair[] { const pairs: ObservationPair[] = []; for (const group of groups) { - const first = group.observationsByHarness.get(firstHarness) ?? []; - const second = group.observationsByHarness.get(secondHarness) ?? []; - if (first.length === 1 && second.length === 1) pairs.push({ first: first[0], second: second[0] }); + const baseline = group.observationsByHarness.get(baselineHarness) ?? []; + const candidate = group.observationsByHarness.get(candidateHarness) ?? []; + if (baseline.length === 1 && candidate.length === 1) { + pairs.push({ baseline: baseline[0], candidate: candidate[0] }); + } } return pairs; } @@ -216,78 +220,82 @@ function summarizeMetric( select: (observation: HarnessObservation) => number | undefined, totalPairs: number, ): PairedMetricSummary { - const firstValues: number[] = []; - const secondValues: number[] = []; - for (const { first, second } of pairs) { - if (first.outcome !== "scored" || second.outcome !== "scored") continue; - const firstValue = select(first); - const secondValue = select(second); + const baselineValues: number[] = []; + const candidateValues: number[] = []; + for (const { baseline, candidate } of pairs) { + if (baseline.outcome !== "scored" || candidate.outcome !== "scored") continue; + const baselineValue = select(baseline); + const candidateValue = select(candidate); if ( - firstValue === undefined || - secondValue === undefined || - !Number.isFinite(firstValue) || - !Number.isFinite(secondValue) + baselineValue === undefined || + candidateValue === undefined || + !Number.isFinite(baselineValue) || + !Number.isFinite(candidateValue) ) { continue; } - firstValues.push(firstValue); - secondValues.push(secondValue); + baselineValues.push(baselineValue); + candidateValues.push(candidateValue); } - const firstMean = mean(firstValues); - const secondMean = mean(secondValues); + const baselineMean = mean(baselineValues); + const candidateMean = mean(candidateValues); return { totalPairs, - eligiblePairs: firstValues.length, - firstMean, - secondMean, - meanDelta: firstMean === null || secondMean === null ? null : preciseDifference(secondMean, firstMean), + eligiblePairs: baselineValues.length, + baselineMean, + candidateMean, + meanDelta: + baselineMean === null || candidateMean === null ? null : preciseDifference(candidateMean, baselineMean), }; } function summarizeCorrectness(pairs: readonly ObservationPair[], totalPairs: number): CorrectnessLiftSummary { let eligiblePairs = 0; - let firstPasses = 0; - let secondPasses = 0; - let firstWins = 0; - let secondWins = 0; + let baselinePasses = 0; + let candidatePasses = 0; + let baselineWins = 0; + let candidateWins = 0; let ties = 0; - for (const { first, second } of pairs) { - if (first.outcome !== "scored" || second.outcome !== "scored") continue; + for (const { baseline, candidate } of pairs) { + if (baseline.outcome !== "scored" || candidate.outcome !== "scored") continue; eligiblePairs += 1; - const firstPassed = first.score >= 1; - const secondPassed = second.score >= 1; - if (firstPassed) firstPasses += 1; - if (secondPassed) secondPasses += 1; - if (firstPassed === secondPassed) ties += 1; - else if (firstPassed) firstWins += 1; - else secondWins += 1; + const baselinePassed = baseline.score >= 1; + const candidatePassed = candidate.score >= 1; + if (baselinePassed) baselinePasses += 1; + if (candidatePassed) candidatePasses += 1; + if (baselinePassed === candidatePassed) ties += 1; + else if (baselinePassed) baselineWins += 1; + else candidateWins += 1; } - const firstPassRate = eligiblePairs === 0 ? null : firstPasses / eligiblePairs; - const secondPassRate = eligiblePairs === 0 ? null : secondPasses / eligiblePairs; + const baselinePassRate = eligiblePairs === 0 ? null : baselinePasses / eligiblePairs; + const candidatePassRate = eligiblePairs === 0 ? null : candidatePasses / eligiblePairs; return { totalPairs, eligiblePairs, - firstPassRate, - secondPassRate, - lift: firstPassRate === null || secondPassRate === null ? null : preciseDifference(secondPassRate, firstPassRate), - firstWins, - secondWins, + baselinePassRate, + candidatePassRate, + lift: + baselinePassRate === null || candidatePassRate === null + ? null + : preciseDifference(candidatePassRate, baselinePassRate), + baselineWins, + candidateWins, ties, }; } function compareHarnesses( - firstHarness: HarnessDescriptor, - secondHarness: HarnessDescriptor, + baseline: HarnessDescriptor, + candidate: HarnessDescriptor, groups: readonly ObservationGroup[], ): HarnessPairComparison { - const pairs = pairObservations(groups, firstHarness.name, secondHarness.name); + const pairs = pairObservations(groups, baseline.name, candidate.name); return { - firstHarness: firstHarness.name, - secondHarness: secondHarness.name, + baseline: baseline.name, + candidate: candidate.name, correctness: summarizeCorrectness(pairs, groups.length), totalTokens: summarizeMetric(pairs, ({ totalTokens }) => totalTokens, groups.length), totalMs: summarizeMetric(pairs, ({ totalMs }) => totalMs, groups.length), @@ -302,10 +310,11 @@ export function summarizeHarnessComparisons(observations: readonly HarnessObserv left.localeCompare(right), )) { const harnesses = orderedHarnesses(data); + const candidates = orderedCandidates(data); const groups = orderedGroups(data); evalSets.push({ evalSet, - comparisons: allPairs(harnesses).map(([first, second]) => compareHarnesses(first, second, groups)), + comparisons: candidates.map((candidate) => compareHarnesses(data.baseline, candidate, groups)), }); diagnostics.push(...collectDiagnostics(harnesses, groups)); } @@ -358,11 +367,11 @@ function formatMetric( metric.eligiblePairs === 0 || metric.eligiblePairs === comparisonPairs ? "" : ` ${formatCoverage(metric.eligiblePairs, metric.totalPairs)}`; - if (metric.firstMean === null || metric.secondMean === null || metric.meanDelta === null) { + if (metric.baselineMean === null || metric.candidateMean === null || metric.meanDelta === null) { return formatReportLine(label, `${styleText("yellow", "unavailable")}${coverage}`); } const delta = colorDelta(metric.meanDelta, `delta ${formatDelta(metric.meanDelta)}`, false); - const values = styleText("gray", `(${formatValue(metric.secondMean)}, ${formatValue(metric.firstMean)})`); + const values = styleText("gray", `(${formatValue(metric.candidateMean)}, ${formatValue(metric.baselineMean)})`); return formatReportLine(label, `${delta} ${values}${coverage}`); } @@ -371,12 +380,14 @@ export function formatHarnessComparisonReport(report: HarnessComparisonReport): const lines = [styleText("bold", "Eval Comparisons")]; for (const evalSet of report.evalSets) { lines.push(` ${evalSet.evalSet}`); - for (const comparison of evalSet.comparisons) { + for (const [index, comparison] of evalSet.comparisons.entries()) { + if (index > 0) lines.push(""); const { correctness } = comparison; + lines.push(formatReportLine("Baseline", comparison.baseline)); lines.push( formatReportLine( - "Harnesses", - `${comparison.secondHarness}, ${comparison.firstHarness} ${formatCoverage(correctness.eligiblePairs, correctness.totalPairs)}`, + "Candidate", + `${comparison.candidate} ${formatCoverage(correctness.eligiblePairs, correctness.totalPairs)}`, ), ); if (correctness.lift === null) { @@ -386,7 +397,7 @@ export function formatHarnessComparisonReport(report: HarnessComparisonReport): const delta = colorDelta(lift, `delta ${formatSigned(lift, 1)} pp`, true); const values = styleText( "gray", - `(${formatPercentage(correctness.secondPassRate)}, ${formatPercentage(correctness.firstPassRate)})`, + `(${formatPercentage(correctness.candidatePassRate)}, ${formatPercentage(correctness.baselinePassRate)})`, ); lines.push(formatReportLine("Pass rate", `${delta} ${values}`)); } diff --git a/packages/evals/test/vitest-evals/harness-table.test.ts b/packages/evals/test/vitest-evals/harness-table.test.ts index 9765e7885..db77f9d40 100644 --- a/packages/evals/test/vitest-evals/harness-table.test.ts +++ b/packages/evals/test/vitest-evals/harness-table.test.ts @@ -45,7 +45,10 @@ function createFakeHarness(name: string) { const harnessTable = evalHarnessTable( "local multi-harness eval", - [createFakeHarness("withoutSkill"), createFakeHarness("withSkill")], + { + baseline: createFakeHarness("withoutSkill"), + candidates: [createFakeHarness("withSkill")], + }, { repetitions: 2, seed: 42 }, ); @@ -61,6 +64,16 @@ describe("evalHarnessTable", () => { ]); }); + it("accepts a singular candidate", () => { + const rows = evalHarnessTable( + "singular candidate", + { baseline: createFakeHarness("baseline"), candidate: createFakeHarness("candidate") }, + { seed: 42 }, + ); + + expect(rows.map(({ name }) => name)).toEqual(["baseline", "candidate"]); + }); + it("attaches iteration metadata to every wrapped harness run", async () => { for (const row of harnessTable) { const artifacts: HarnessContext["artifacts"] = {}; @@ -78,7 +91,8 @@ describe("evalHarnessTable", () => { evalSet: "local multi-harness eval", groupKey: deriveEvalGroupKey({ id: "first" }, row.repetition, 42), harness: row.name, - harnesses: ["withoutSkill", "withSkill"], + baseline: "withoutSkill", + candidates: ["withSkill"], repetition: row.repetition, seed: 42, plannedOrder: row.plannedOrder, diff --git a/packages/evals/test/vitest-evals/summary.test.ts b/packages/evals/test/vitest-evals/summary.test.ts index e9ee18a1e..077a1d2b1 100644 --- a/packages/evals/test/vitest-evals/summary.test.ts +++ b/packages/evals/test/vitest-evals/summary.test.ts @@ -13,7 +13,8 @@ function observation( testName: string, result: ObservationResult, metrics: Pick = {}, - harnesses: string[] = ["without-tools", "with-tools"], + baseline = "without-tools", + candidates: string[] = ["with-tools"], ): HarnessObservation { const base = { evalSet: "tool access", @@ -21,7 +22,8 @@ function observation( testName, file: "src/tool-access.eval.ts", harness, - harnesses, + baseline, + candidates, repetition: 1, seed: 42, ...metrics, @@ -52,37 +54,37 @@ describe("summarizeHarnessComparisons", () => { expect(report.evalSets).toHaveLength(1); expect(report.evalSets[0]?.comparisons).toEqual([ expect.objectContaining({ - firstHarness: "without-tools", - secondHarness: "with-tools", + baseline: "without-tools", + candidate: "with-tools", correctness: { totalPairs: 2, eligiblePairs: 2, - firstPassRate: 0.5, - secondPassRate: 1, + baselinePassRate: 0.5, + candidatePassRate: 1, lift: 0.5, - firstWins: 0, - secondWins: 1, + baselineWins: 0, + candidateWins: 1, ties: 1, }, totalTokens: { totalPairs: 2, eligiblePairs: 2, - firstMean: 150, - secondMean: 150, + baselineMean: 150, + candidateMean: 150, meanDelta: 0, }, totalMs: { totalPairs: 2, eligiblePairs: 1, - firstMean: 1000, - secondMean: 800, + baselineMean: 1000, + candidateMean: 800, meanDelta: -200, }, estimatedCostUsd: { totalPairs: 2, eligiblePairs: 1, - firstMean: 0.01, - secondMean: 0.02, + baselineMean: 0.01, + candidateMean: 0.02, meanDelta: 0.01, }, }), @@ -101,18 +103,18 @@ describe("summarizeHarnessComparisons", () => { expect(comparison?.correctness).toEqual({ totalPairs: 2, eligiblePairs: 1, - firstPassRate: 0, - secondPassRate: 1, + baselinePassRate: 0, + candidatePassRate: 1, lift: 1, - firstWins: 0, - secondWins: 1, + baselineWins: 0, + candidateWins: 1, ties: 0, }); expect(comparison?.totalTokens).toEqual({ totalPairs: 2, eligiblePairs: 0, - firstMean: null, - secondMean: null, + baselineMean: null, + candidateMean: null, meanDelta: null, }); expect(report.diagnostics).toContainEqual( @@ -166,20 +168,17 @@ describe("summarizeHarnessComparisons", () => { ]); }); - it("creates every pair in declared harness order", () => { - const harnesses = ["first", "second", "third"]; + it("compares each candidate with the declared baseline", () => { + const candidates = ["second", "third"]; const report = summarizeHarnessComparisons([ - observation("first", "input", "passed", {}, harnesses), - observation("second", "input", "passed", {}, harnesses), - observation("third", "input", "passed", {}, harnesses), + observation("first", "input", "passed", {}, "first", candidates), + observation("second", "input", "passed", {}, "first", candidates), + observation("third", "input", "passed", {}, "first", candidates), ]); - expect( - report.evalSets[0]?.comparisons.map(({ firstHarness, secondHarness }) => [firstHarness, secondHarness]), - ).toEqual([ + expect(report.evalSets[0]?.comparisons.map(({ baseline, candidate }) => [baseline, candidate])).toEqual([ ["first", "second"], ["first", "third"], - ["second", "third"], ]); }); @@ -198,15 +197,15 @@ describe("summarizeHarnessComparisons", () => { }); it("reports duplicate and unscorable observations once across multiple harness pairs", () => { - const harnesses = ["first", "second", "third"]; + const candidates = ["second", "third"]; const report = summarizeHarnessComparisons([ - observation("first", "duplicate", "passed", {}, harnesses), - observation("first", "duplicate", "failed", {}, harnesses), - observation("second", "duplicate", "passed", {}, harnesses), - observation("third", "duplicate", "passed", {}, harnesses), - observation("first", "skipped", "skipped", {}, harnesses), - observation("second", "skipped", "passed", {}, harnesses), - observation("third", "skipped", "passed", {}, harnesses), + observation("first", "duplicate", "passed", {}, "first", candidates), + observation("first", "duplicate", "failed", {}, "first", candidates), + observation("second", "duplicate", "passed", {}, "first", candidates), + observation("third", "duplicate", "passed", {}, "first", candidates), + observation("first", "skipped", "skipped", {}, "first", candidates), + observation("second", "skipped", "passed", {}, "first", candidates), + observation("third", "skipped", "passed", {}, "first", candidates), ]); expect(report.diagnostics.filter(({ reason }) => reason === "duplicate-observation")).toEqual([ @@ -225,7 +224,8 @@ describe("summarizeHarnessComparisons", () => { const formatted = stripVTControlCharacters(formatHarnessComparisonReport(report)); expect(formatted).toContain("Eval Comparisons"); - expect(formatted).toContain("Harnesses with-tools, without-tools (1/1 pairs)"); + expect(formatted).toContain(" Baseline without-tools"); + expect(formatted).toContain("Candidate with-tools (1/1 pairs)"); expect(formatted).toContain("Pass rate delta +100.0 pp (100.0%, 0.0%)"); expect(formatted).toContain(" Tokens unavailable"); expect(formatted).toContain(" Latency delta -4159.5ms (30694.2ms, 34853.7ms)");