sscoble/pi-mono
1
0
Fork 0
mirror of https://github.com/badlogic/pi-mono.git synced 2026-11-05 23:37:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

fix(coding-agent): make eval comparison direction explicit

Browse source
This commit is contained in:
Christian Klotz 2026-07-30 10:58:07 +03:00
parent ac033d1865
commit fce7445b1c
No known key found for this signature in database
7 changed files with 205 additions and 145 deletions
Download patch file Download diff file Expand all files Collapse all files

17
packages/evals/README.md
View file

@ -117,7 +117,10 @@ const TargetTaskJudge = createJudge<string, string>("TargetTaskJudge", ({ output
const harnessTable = evalHarnessTable(
"target skill effectiveness",
[withoutTargetSkillHarness, withTargetSkillHarness],
{
baseline: withoutTargetSkillHarness,
candidate: withTargetSkillHarness,
},
{
repetitions: 6,
seed: 42,
@ -142,12 +145,12 @@ registers that snapshot against the explicit Vitest test task before reporters r
Harness names must be stable and unique within an eval set. The grouping key combines repetition and seed with a
non-empty string `input.id` when available, otherwise with a SHA-256 hash of strict canonical JSON input. Execution order
is seeded and randomized, while harness declaration order determines comparison direction. For each matched input and
repetition, the reporter computes pass-rate lift from each run's recorded average judge score, treating a score of at
least `1` as passing. The lift is the later-declared harness's pass rate minus the earlier-declared harness's pass rate,
in percentage points. Missing judge scores are reported as incomplete observations. Tokens, latency, and estimated cost
remain separate paired deltas; missing telemetry remains unavailable. Sets
with more than two harnesses report every pair.
is seeded and randomized independently of comparison direction. Use `candidate` for one treatment or `candidates` for
multiple treatments. Each candidate is compared only with the declared baseline. For each matched input and repetition,
the reporter computes pass-rate lift from each run's recorded average judge score, treating a score of at least `1` as
passing. Lift is the candidate pass rate minus the baseline pass rate, in percentage points. Missing judge scores are
reported as incomplete observations. Tokens, latency, and estimated cost remain separate candidate-minus-baseline paired
deltas; missing telemetry remains unavailable.
See the [`skill-eval-harness`](https://github.com/adewale/skill-eval-harness/) guidance for comparative-eval methodology,
repetition strategy, trustworthy judges, and telemetry interpretation.

8
packages/evals/src/extensions.eval.ts
View file

@ -99,10 +99,10 @@ const ExtensionAuthoringJudge = createJudge<PiCodingAgentInput, ExtensionAuthori
const extensionHarnessTable = evalHarnessTable(
"Pi extension authoring system prompt",
[
createExtensionAuthoringHarness("system-prompt-without-docs", excludeGuidelinesAndDocumentation),
createExtensionAuthoringHarness("default-system-prompt", prepareDefaultPromptOverride),
],
{
baseline: createExtensionAuthoringHarness("system-prompt-without-docs", excludeGuidelinesAndDocumentation),
candidate: createExtensionAuthoringHarness("default-system-prompt", prepareDefaultPromptOverride),
},
{ seed: 42 },
);

51
packages/evals/src/vitest-evals/harness-table.ts
View file

@ -14,7 +14,8 @@ export type EvalHarnessIterationArtifact = {
evalSet: string;
groupKey: string;
harness: string;
harnesses: string[];
baseline: string;
candidates: string[];
repetition: number;
seed: number;
plannedOrder: number;
@ -33,27 +34,42 @@ export type EvalHarnessTableOptions = {
seed?: number;
};
export type EvalHarnessTablePair<TInput, TOutput extends JsonValue | undefined> = {
baseline: Harness<TInput, TOutput>;
candidate: Harness<TInput, TOutput>;
};
export type EvalHarnessTableCandidates<TInput, TOutput extends JsonValue | undefined> = {
baseline: Harness<TInput, TOutput>;
candidates: readonly Harness<TInput, TOutput>[];
};
type EvalHarnessTableComparison<TInput, TOutput extends JsonValue | undefined> =
| EvalHarnessTablePair<TInput, TOutput>
| EvalHarnessTableCandidates<TInput, TOutput>;
type EvalHarnessIterationPlan = Omit<EvalHarnessIterationArtifact, "groupKey">;
export function parseEvalHarnessIterationArtifact(
value: JsonValue | undefined,
): EvalHarnessIterationArtifact | undefined {
if (value === null || value === undefined || typeof value !== "object" || Array.isArray(value)) return undefined;
const { schemaVersion, evalSet, groupKey, harness, harnesses, repetition, seed, plannedOrder } = value;
const { schemaVersion, evalSet, groupKey, harness, baseline, candidates, repetition, seed, plannedOrder } = value;
if (
schemaVersion !== 1 ||
typeof evalSet !== "string" ||
typeof groupKey !== "string" ||
typeof harness !== "string" ||
!Array.isArray(harnesses) ||
!harnesses.every((name): name is string => typeof name === "string") ||
typeof baseline !== "string" ||
!Array.isArray(candidates) ||
!candidates.every((name): name is string => typeof name === "string") ||
typeof repetition !== "number" ||
typeof seed !== "number" ||
typeof plannedOrder !== "number"
) {
return undefined;
}
return { schemaVersion, evalSet, groupKey, harness, harnesses, repetition, seed, plannedOrder };
return { schemaVersion, evalSet, groupKey, harness, baseline, candidates, repetition, seed, plannedOrder };
}
function createSeededRandom(seed: number): () => number {
@ -117,12 +133,14 @@ export function deriveEvalGroupKey(input: unknown, repetition: number, seed: num
function validateOptions<TInput, TOutput extends JsonValue | undefined>(
evalSet: string,
harnesses: readonly Harness<TInput, TOutput>[],
baseline: Harness<TInput, TOutput>,
candidates: readonly Harness<TInput, TOutput>[],
repetitions: number,
seed: number,
): void {
if (!evalSet.trim()) throw new TypeError("evalSet must not be empty.");
if (harnesses.length < 2) throw new TypeError("At least two harnesses are required.");
if (candidates.length === 0) throw new TypeError("At least one candidate harness is required.");
const harnesses = [baseline, ...candidates];
const names = new Set(harnesses.map((harness) => harness.name));
if (names.size !== harnesses.length) throw new TypeError("Harness names must be unique within an eval set.");
if (!Number.isSafeInteger(repetitions) || repetitions < 1) {
@ -162,15 +180,27 @@ function withIterationArtifact<TInput, TOutput extends JsonValue | undefined>(
export function evalHarnessTable<TInput, TOutput extends JsonValue | undefined>(
evalSet: string,
harnesses: readonly Harness<TInput, TOutput>[],
comparison: EvalHarnessTablePair<TInput, TOutput>,
options: EvalHarnessTableOptions,
): EvalHarnessTableRow<TInput, TOutput>[];
export function evalHarnessTable<TInput, TOutput extends JsonValue | undefined>(
evalSet: string,
comparison: EvalHarnessTableCandidates<TInput, TOutput>,
options: EvalHarnessTableOptions,
): EvalHarnessTableRow<TInput, TOutput>[];
export function evalHarnessTable<TInput, TOutput extends JsonValue | undefined>(
evalSet: string,
comparison: EvalHarnessTableComparison<TInput, TOutput>,
options: EvalHarnessTableOptions,
): EvalHarnessTableRow<TInput, TOutput>[] {
const repetitions = options.repetitions ?? 1;
const seed = options.seed ?? randomBytes(4).readUInt32LE();
validateOptions(evalSet, harnesses, repetitions, seed);
const candidates = "candidate" in comparison ? [comparison.candidate] : comparison.candidates;
validateOptions(evalSet, comparison.baseline, candidates, repetitions, seed);
const random = createSeededRandom(seed);
const rows: EvalHarnessTableRow<TInput, TOutput>[] = [];
const harnesses = [comparison.baseline, ...candidates];
let plannedOrder = 1;
for (let repetition = 1; repetition <= repetitions; repetition += 1) {
const randomizedHarnesses = [...harnesses];
@ -186,7 +216,8 @@ export function evalHarnessTable<TInput, TOutput extends JsonValue | undefined>(
schemaVersion: 1,
evalSet,
harness: harness.name,
harnesses: harnesses.map(({ name }) => name),
baseline: comparison.baseline.name,
candidates: candidates.map(({ name }) => name),
repetition,
seed,
plannedOrder,

5
packages/evals/src/vitest-evals/reporter.ts
View file

@ -65,7 +65,8 @@ function collectHarnessObservations(modules: ReadonlyArray<TestModule>): Harness
testName: test.name,
file: module.relativeModuleId,
harness: iteration.harness,
harnesses: iteration.harnesses,
baseline: iteration.baseline,
candidates: iteration.candidates,
repetition: iteration.repetition,
seed: iteration.seed,
...(run.usage.totalTokens === undefined ? {} : { totalTokens: run.usage.totalTokens }),
@ -101,7 +102,7 @@ export default class EvalHarnessReporter implements Reporter {
reason: TestRunEndReason,
): void {
if (reason === "interrupted") {
this.vitest?.logger.log("\nHarness comparisons unavailable: test run interrupted.");
this.vitest?.logger.log("\nEval comparisons unavailable: test run interrupted.");
return;
}
const report = summarizeHarnessComparisons(collectHarnessObservations(modules));

175
packages/evals/src/vitest-evals/summary.ts
View file

@ -8,7 +8,8 @@ type HarnessObservationBase = {
testName: string;
file: string;
harness: string;
harnesses: string[];
baseline: string;
candidates: string[];
repetition: number;
seed: number;
totalTokens?: number;
@ -22,25 +23,25 @@ export type HarnessObservation = HarnessObservationBase &
export type PairedMetricSummary = {
totalPairs: number;
eligiblePairs: number;
firstMean: number | null;
secondMean: number | null;
baselineMean: number | null;
candidateMean: number | null;
meanDelta: number | null;
};
export type CorrectnessLiftSummary = {
totalPairs: number;
eligiblePairs: number;
firstPassRate: number | null;
secondPassRate: number | null;
baselinePassRate: number | null;
candidatePassRate: number | null;
lift: number | null;
firstWins: number;
secondWins: number;
baselineWins: number;
candidateWins: number;
ties: number;
};
export type HarnessPairComparison = {
firstHarness: string;
secondHarness: string;
baseline: string;
candidate: string;
correctness: CorrectnessLiftSummary;
totalTokens: PairedMetricSummary;
totalMs: PairedMetricSummary;
@ -85,13 +86,14 @@ type ObservationGroup = {
};
type EvalSetData = {
harnessesByName: Map<string, HarnessDescriptor>;
baseline: HarnessDescriptor;
candidatesByName: Map<string, HarnessDescriptor>;
groupsByKey: Map<string, ObservationGroup>;
};
type ObservationPair = {
first: HarnessObservation;
second: HarnessObservation;
baseline: HarnessObservation;
candidate: HarnessObservation;
};
function getOrCreate<K, V extends object>(map: Map<K, V>, key: K, create: () => V): V {
@ -110,27 +112,18 @@ function preciseDifference(left: number, right: number): number {
return Number((left - right).toPrecision(15));
}
function allPairs<T>(values: readonly T[]): Array<[T, T]> {
const pairs: Array<[T, T]> = [];
for (let first = 0; first < values.length; first += 1) {
for (let second = first + 1; second < values.length; second += 1) {
pairs.push([values[first], values[second]]);
}
}
return pairs;
}
function groupObservations(observations: readonly HarnessObservation[]): Map<string, EvalSetData> {
const evalSets = new Map<string, EvalSetData>();
for (const observation of observations) {
const evalSet = getOrCreate(evalSets, observation.evalSet, () => ({
harnessesByName: new Map(),
baseline: { name: observation.baseline, index: 0 },
candidatesByName: new Map(),
groupsByKey: new Map(),
}));
for (const [index, name] of observation.harnesses.entries()) {
const existing = evalSet.harnessesByName.get(name);
if (!existing || index < existing.index) evalSet.harnessesByName.set(name, { name, index });
for (const [index, name] of observation.candidates.entries()) {
const existing = evalSet.candidatesByName.get(name);
if (!existing || index < existing.index) evalSet.candidatesByName.set(name, { name, index });
}
const group = getOrCreate(
@ -152,7 +145,16 @@ function groupObservations(observations: readonly HarnessObservation[]): Map<str
}
function orderedHarnesses(evalSet: EvalSetData): HarnessDescriptor[] {
return [...evalSet.harnessesByName.values()].sort(
return [
evalSet.baseline,
...[...evalSet.candidatesByName.values()].sort(
(left, right) => left.index - right.index || left.name.localeCompare(right.name),
),
];
}
function orderedCandidates(evalSet: EvalSetData): HarnessDescriptor[] {
return [...evalSet.candidatesByName.values()].sort(
(left, right) => left.index - right.index || left.name.localeCompare(right.name),
);
}
@ -199,14 +201,16 @@ function collectDiagnostics(
function pairObservations(
groups: readonly ObservationGroup[],
firstHarness: string,
secondHarness: string,
baselineHarness: string,
candidateHarness: string,
): ObservationPair[] {
const pairs: ObservationPair[] = [];
for (const group of groups) {
const first = group.observationsByHarness.get(firstHarness) ?? [];
const second = group.observationsByHarness.get(secondHarness) ?? [];
if (first.length === 1 && second.length === 1) pairs.push({ first: first[0], second: second[0] });
const baseline = group.observationsByHarness.get(baselineHarness) ?? [];
const candidate = group.observationsByHarness.get(candidateHarness) ?? [];
if (baseline.length === 1 && candidate.length === 1) {
pairs.push({ baseline: baseline[0], candidate: candidate[0] });
}
}
return pairs;
}
@ -216,78 +220,82 @@ function summarizeMetric(
select: (observation: HarnessObservation) => number | undefined,
totalPairs: number,
): PairedMetricSummary {
const firstValues: number[] = [];
const secondValues: number[] = [];
for (const { first, second } of pairs) {
if (first.outcome !== "scored" || second.outcome !== "scored") continue;
const firstValue = select(first);
const secondValue = select(second);
const baselineValues: number[] = [];
const candidateValues: number[] = [];
for (const { baseline, candidate } of pairs) {
if (baseline.outcome !== "scored" || candidate.outcome !== "scored") continue;
const baselineValue = select(baseline);
const candidateValue = select(candidate);
if (
firstValue === undefined ||
secondValue === undefined ||
!Number.isFinite(firstValue) ||
!Number.isFinite(secondValue)
baselineValue === undefined ||
candidateValue === undefined ||
!Number.isFinite(baselineValue) ||
!Number.isFinite(candidateValue)
) {
continue;
}
firstValues.push(firstValue);
secondValues.push(secondValue);
baselineValues.push(baselineValue);
candidateValues.push(candidateValue);
}
const firstMean = mean(firstValues);
const secondMean = mean(secondValues);
const baselineMean = mean(baselineValues);
const candidateMean = mean(candidateValues);
return {
totalPairs,
eligiblePairs: firstValues.length,
firstMean,
secondMean,
meanDelta: firstMean === null || secondMean === null ? null : preciseDifference(secondMean, firstMean),
eligiblePairs: baselineValues.length,
baselineMean,
candidateMean,
meanDelta:
baselineMean === null || candidateMean === null ? null : preciseDifference(candidateMean, baselineMean),
};
}
function summarizeCorrectness(pairs: readonly ObservationPair[], totalPairs: number): CorrectnessLiftSummary {
let eligiblePairs = 0;
let firstPasses = 0;
let secondPasses = 0;
let firstWins = 0;
let secondWins = 0;
let baselinePasses = 0;
let candidatePasses = 0;
let baselineWins = 0;
let candidateWins = 0;
let ties = 0;
for (const { first, second } of pairs) {
if (first.outcome !== "scored" || second.outcome !== "scored") continue;
for (const { baseline, candidate } of pairs) {
if (baseline.outcome !== "scored" || candidate.outcome !== "scored") continue;
eligiblePairs += 1;
const firstPassed = first.score >= 1;
const secondPassed = second.score >= 1;
if (firstPassed) firstPasses += 1;
if (secondPassed) secondPasses += 1;
if (firstPassed === secondPassed) ties += 1;
else if (firstPassed) firstWins += 1;
else secondWins += 1;
const baselinePassed = baseline.score >= 1;
const candidatePassed = candidate.score >= 1;
if (baselinePassed) baselinePasses += 1;
if (candidatePassed) candidatePasses += 1;
if (baselinePassed === candidatePassed) ties += 1;
else if (baselinePassed) baselineWins += 1;
else candidateWins += 1;
}
const firstPassRate = eligiblePairs === 0 ? null : firstPasses / eligiblePairs;
const secondPassRate = eligiblePairs === 0 ? null : secondPasses / eligiblePairs;
const baselinePassRate = eligiblePairs === 0 ? null : baselinePasses / eligiblePairs;
const candidatePassRate = eligiblePairs === 0 ? null : candidatePasses / eligiblePairs;
return {
totalPairs,
eligiblePairs,
firstPassRate,
secondPassRate,
lift: firstPassRate === null || secondPassRate === null ? null : preciseDifference(secondPassRate, firstPassRate),
firstWins,
secondWins,
baselinePassRate,
candidatePassRate,
lift:
baselinePassRate === null || candidatePassRate === null
? null
: preciseDifference(candidatePassRate, baselinePassRate),
baselineWins,
candidateWins,
ties,
};
}
function compareHarnesses(
firstHarness: HarnessDescriptor,
secondHarness: HarnessDescriptor,
baseline: HarnessDescriptor,
candidate: HarnessDescriptor,
groups: readonly ObservationGroup[],
): HarnessPairComparison {
const pairs = pairObservations(groups, firstHarness.name, secondHarness.name);
const pairs = pairObservations(groups, baseline.name, candidate.name);
return {
firstHarness: firstHarness.name,
secondHarness: secondHarness.name,
baseline: baseline.name,
candidate: candidate.name,
correctness: summarizeCorrectness(pairs, groups.length),
totalTokens: summarizeMetric(pairs, ({ totalTokens }) => totalTokens, groups.length),
totalMs: summarizeMetric(pairs, ({ totalMs }) => totalMs, groups.length),
@ -302,10 +310,11 @@ export function summarizeHarnessComparisons(observations: readonly HarnessObserv
left.localeCompare(right),
)) {
const harnesses = orderedHarnesses(data);
const candidates = orderedCandidates(data);
const groups = orderedGroups(data);
evalSets.push({
evalSet,
comparisons: allPairs(harnesses).map(([first, second]) => compareHarnesses(first, second, groups)),
comparisons: candidates.map((candidate) => compareHarnesses(data.baseline, candidate, groups)),
});
diagnostics.push(...collectDiagnostics(harnesses, groups));
}
@ -358,11 +367,11 @@ function formatMetric(
metric.eligiblePairs === 0 || metric.eligiblePairs === comparisonPairs
? ""
: ` ${formatCoverage(metric.eligiblePairs, metric.totalPairs)}`;
if (metric.firstMean === null || metric.secondMean === null || metric.meanDelta === null) {
if (metric.baselineMean === null || metric.candidateMean === null || metric.meanDelta === null) {
return formatReportLine(label, `${styleText("yellow", "unavailable")}${coverage}`);
}
const delta = colorDelta(metric.meanDelta, `delta ${formatDelta(metric.meanDelta)}`, false);
const values = styleText("gray", `(${formatValue(metric.secondMean)}, ${formatValue(metric.firstMean)})`);
const values = styleText("gray", `(${formatValue(metric.candidateMean)}, ${formatValue(metric.baselineMean)})`);
return formatReportLine(label, `${delta} ${values}${coverage}`);
}
@ -371,12 +380,14 @@ export function formatHarnessComparisonReport(report: HarnessComparisonReport):
const lines = [styleText("bold", "Eval Comparisons")];
for (const evalSet of report.evalSets) {
lines.push(` ${evalSet.evalSet}`);
for (const comparison of evalSet.comparisons) {
for (const [index, comparison] of evalSet.comparisons.entries()) {
if (index > 0) lines.push("");
const { correctness } = comparison;
lines.push(formatReportLine("Baseline", comparison.baseline));
lines.push(
formatReportLine(
"Harnesses",
`${comparison.secondHarness}, ${comparison.firstHarness} ${formatCoverage(correctness.eligiblePairs, correctness.totalPairs)}`,
"Candidate",
`${comparison.candidate} ${formatCoverage(correctness.eligiblePairs, correctness.totalPairs)}`,
),
);
if (correctness.lift === null) {
@ -386,7 +397,7 @@ export function formatHarnessComparisonReport(report: HarnessComparisonReport):
const delta = colorDelta(lift, `delta ${formatSigned(lift, 1)} pp`, true);
const values = styleText(
"gray",
`(${formatPercentage(correctness.secondPassRate)}, ${formatPercentage(correctness.firstPassRate)})`,
`(${formatPercentage(correctness.candidatePassRate)}, ${formatPercentage(correctness.baselinePassRate)})`,
);
lines.push(formatReportLine("Pass rate", `${delta} ${values}`));
}

18
packages/evals/test/vitest-evals/harness-table.test.ts
View file

@ -45,7 +45,10 @@ function createFakeHarness(name: string) {
const harnessTable = evalHarnessTable(
"local multi-harness eval",
[createFakeHarness("withoutSkill"), createFakeHarness("withSkill")],
{
baseline: createFakeHarness("withoutSkill"),
candidates: [createFakeHarness("withSkill")],
},
{ repetitions: 2, seed: 42 },
);
@ -61,6 +64,16 @@ describe("evalHarnessTable", () => {
]);
});
it("accepts a singular candidate", () => {
const rows = evalHarnessTable(
"singular candidate",
{ baseline: createFakeHarness("baseline"), candidate: createFakeHarness("candidate") },
{ seed: 42 },
);
expect(rows.map(({ name }) => name)).toEqual(["baseline", "candidate"]);
});
it("attaches iteration metadata to every wrapped harness run", async () => {
for (const row of harnessTable) {
const artifacts: HarnessContext["artifacts"] = {};
@ -78,7 +91,8 @@ describe("evalHarnessTable", () => {
evalSet: "local multi-harness eval",
groupKey: deriveEvalGroupKey({ id: "first" }, row.repetition, 42),
harness: row.name,
harnesses: ["withoutSkill", "withSkill"],
baseline: "withoutSkill",
candidates: ["withSkill"],
repetition: row.repetition,
seed: 42,
plannedOrder: row.plannedOrder,

76
packages/evals/test/vitest-evals/summary.test.ts
View file

@ -13,7 +13,8 @@ function observation(
testName: string,
result: ObservationResult,
metrics: Pick<HarnessObservation, "totalTokens" | "totalMs" | "estimatedCostUsd"> = {},
harnesses: string[] = ["without-tools", "with-tools"],
baseline = "without-tools",
candidates: string[] = ["with-tools"],
): HarnessObservation {
const base = {
evalSet: "tool access",
@ -21,7 +22,8 @@ function observation(
testName,
file: "src/tool-access.eval.ts",
harness,
harnesses,
baseline,
candidates,
repetition: 1,
seed: 42,
...metrics,
@ -52,37 +54,37 @@ describe("summarizeHarnessComparisons", () => {
expect(report.evalSets).toHaveLength(1);
expect(report.evalSets[0]?.comparisons).toEqual([
expect.objectContaining({
firstHarness: "without-tools",
secondHarness: "with-tools",
baseline: "without-tools",
candidate: "with-tools",
correctness: {
totalPairs: 2,
eligiblePairs: 2,
firstPassRate: 0.5,
secondPassRate: 1,
baselinePassRate: 0.5,
candidatePassRate: 1,
lift: 0.5,
firstWins: 0,
secondWins: 1,
baselineWins: 0,
candidateWins: 1,
ties: 1,
},
totalTokens: {
totalPairs: 2,
eligiblePairs: 2,
firstMean: 150,
secondMean: 150,
baselineMean: 150,
candidateMean: 150,
meanDelta: 0,
},
totalMs: {
totalPairs: 2,
eligiblePairs: 1,
firstMean: 1000,
secondMean: 800,
baselineMean: 1000,
candidateMean: 800,
meanDelta: -200,
},
estimatedCostUsd: {
totalPairs: 2,
eligiblePairs: 1,
firstMean: 0.01,
secondMean: 0.02,
baselineMean: 0.01,
candidateMean: 0.02,
meanDelta: 0.01,
},
}),
@ -101,18 +103,18 @@ describe("summarizeHarnessComparisons", () => {
expect(comparison?.correctness).toEqual({
totalPairs: 2,
eligiblePairs: 1,
firstPassRate: 0,
secondPassRate: 1,
baselinePassRate: 0,
candidatePassRate: 1,
lift: 1,
firstWins: 0,
secondWins: 1,
baselineWins: 0,
candidateWins: 1,
ties: 0,
});
expect(comparison?.totalTokens).toEqual({
totalPairs: 2,
eligiblePairs: 0,
firstMean: null,
secondMean: null,
baselineMean: null,
candidateMean: null,
meanDelta: null,
});
expect(report.diagnostics).toContainEqual(
@ -166,20 +168,17 @@ describe("summarizeHarnessComparisons", () => {
]);
});
it("creates every pair in declared harness order", () => {
const harnesses = ["first", "second", "third"];
it("compares each candidate with the declared baseline", () => {
const candidates = ["second", "third"];
const report = summarizeHarnessComparisons([
observation("first", "input", "passed", {}, harnesses),
observation("second", "input", "passed", {}, harnesses),
observation("third", "input", "passed", {}, harnesses),
observation("first", "input", "passed", {}, "first", candidates),
observation("second", "input", "passed", {}, "first", candidates),
observation("third", "input", "passed", {}, "first", candidates),
]);
expect(
report.evalSets[0]?.comparisons.map(({ firstHarness, secondHarness }) => [firstHarness, secondHarness]),
).toEqual([
expect(report.evalSets[0]?.comparisons.map(({ baseline, candidate }) => [baseline, candidate])).toEqual([
["first", "second"],
["first", "third"],
["second", "third"],
]);
});
@ -198,15 +197,15 @@ describe("summarizeHarnessComparisons", () => {
});
it("reports duplicate and unscorable observations once across multiple harness pairs", () => {
const harnesses = ["first", "second", "third"];
const candidates = ["second", "third"];
const report = summarizeHarnessComparisons([
observation("first", "duplicate", "passed", {}, harnesses),
observation("first", "duplicate", "failed", {}, harnesses),
observation("second", "duplicate", "passed", {}, harnesses),
observation("third", "duplicate", "passed", {}, harnesses),
observation("first", "skipped", "skipped", {}, harnesses),
observation("second", "skipped", "passed", {}, harnesses),
observation("third", "skipped", "passed", {}, harnesses),
observation("first", "duplicate", "passed", {}, "first", candidates),
observation("first", "duplicate", "failed", {}, "first", candidates),
observation("second", "duplicate", "passed", {}, "first", candidates),
observation("third", "duplicate", "passed", {}, "first", candidates),
observation("first", "skipped", "skipped", {}, "first", candidates),
observation("second", "skipped", "passed", {}, "first", candidates),
observation("third", "skipped", "passed", {}, "first", candidates),
]);
expect(report.diagnostics.filter(({ reason }) => reason === "duplicate-observation")).toEqual([
@ -225,7 +224,8 @@ describe("summarizeHarnessComparisons", () => {
const formatted = stripVTControlCharacters(formatHarnessComparisonReport(report));
expect(formatted).toContain("Eval Comparisons");
expect(formatted).toContain("Harnesses with-tools, without-tools (1/1 pairs)");
expect(formatted).toContain(" Baseline without-tools");
expect(formatted).toContain("Candidate with-tools (1/1 pairs)");
expect(formatted).toContain("Pass rate delta +100.0 pp (100.0%, 0.0%)");
expect(formatted).toContain(" Tokens unavailable");
expect(formatted).toContain(" Latency delta -4159.5ms (30694.2ms, 34853.7ms)");