fix(coding-agent): streamline comparative eval ergonomics

packages/evals/README.md

@@ -120,8 +120,6 @@ const harnessTable = evalHarnessTable(
 	{
 		baseline: withoutTargetSkillHarness,
 		candidate: withTargetSkillHarness,
-	},
-	{
 		repetitions: 6,
 		seed: 42,
 	},

packages/evals/src/extensions.eval.ts

@@ -97,14 +97,11 @@ const ExtensionAuthoringJudge = createJudge<PiCodingAgentInput, ExtensionAuthori
 	},
 );
 
-const extensionHarnessTable = evalHarnessTable(
-	"Pi extension authoring system prompt",
-	{
-		baseline: createExtensionAuthoringHarness("system-prompt-without-docs", excludeGuidelinesAndDocumentation),
-		candidate: createExtensionAuthoringHarness("default-system-prompt", prepareDefaultPromptOverride),
-	},
-	{ seed: 42 },
-);
+const extensionHarnessTable = evalHarnessTable("Pi extension authoring system prompt", {
+	baseline: createExtensionAuthoringHarness("system-prompt-without-docs", excludeGuidelinesAndDocumentation),
+	candidate: createExtensionAuthoringHarness("default-system-prompt", prepareDefaultPromptOverride),
+	seed: 42,
+});
 
 describe.for(extensionHarnessTable)("$name", ({ harness }) => {
 	describeEval(

packages/evals/src/vitest-evals/harness-table.ts

@@ -29,24 +29,23 @@ export type EvalHarnessTableRow<TInput, TOutput extends JsonValue | undefined> =
 	plannedOrder: number;
 };
 
-export type EvalHarnessTableOptions = {
+export type EvalHarnessTablePairOptions<TInput, TOutput extends JsonValue | undefined> = {
+	baseline: Harness<TInput, TOutput>;
+	candidate: Harness<TInput, TOutput>;
 	repetitions?: number;
 	seed?: number;
 };
 
-export type EvalHarnessTablePair<TInput, TOutput extends JsonValue | undefined> = {
-	baseline: Harness<TInput, TOutput>;
-	candidate: Harness<TInput, TOutput>;
-};
-
-export type EvalHarnessTableCandidates<TInput, TOutput extends JsonValue | undefined> = {
+export type EvalHarnessTableCandidatesOptions<TInput, TOutput extends JsonValue | undefined> = {
 	baseline: Harness<TInput, TOutput>;
 	candidates: readonly Harness<TInput, TOutput>[];
+	repetitions?: number;
+	seed?: number;
 };
 
-type EvalHarnessTableComparison<TInput, TOutput extends JsonValue | undefined> =
-	| EvalHarnessTablePair<TInput, TOutput>
-	| EvalHarnessTableCandidates<TInput, TOutput>;
+export type EvalHarnessTableOptions<TInput, TOutput extends JsonValue | undefined> =
+	| EvalHarnessTablePairOptions<TInput, TOutput>
+	| EvalHarnessTableCandidatesOptions<TInput, TOutput>;
 
 type EvalHarnessIterationPlan = Omit<EvalHarnessIterationArtifact, "groupKey">;
 
@@ -180,27 +179,24 @@ function withIterationArtifact<TInput, TOutput extends JsonValue | undefined>(
 
 export function evalHarnessTable<TInput, TOutput extends JsonValue | undefined>(
 	evalSet: string,
-	comparison: EvalHarnessTablePair<TInput, TOutput>,
-	options: EvalHarnessTableOptions,
+	options: EvalHarnessTablePairOptions<TInput, TOutput>,
 ): EvalHarnessTableRow<TInput, TOutput>[];
 export function evalHarnessTable<TInput, TOutput extends JsonValue | undefined>(
 	evalSet: string,
-	comparison: EvalHarnessTableCandidates<TInput, TOutput>,
-	options: EvalHarnessTableOptions,
+	options: EvalHarnessTableCandidatesOptions<TInput, TOutput>,
 ): EvalHarnessTableRow<TInput, TOutput>[];
 export function evalHarnessTable<TInput, TOutput extends JsonValue | undefined>(
 	evalSet: string,
-	comparison: EvalHarnessTableComparison<TInput, TOutput>,
-	options: EvalHarnessTableOptions,
+	options: EvalHarnessTableOptions<TInput, TOutput>,
 ): EvalHarnessTableRow<TInput, TOutput>[] {
 	const repetitions = options.repetitions ?? 1;
 	const seed = options.seed ?? randomBytes(4).readUInt32LE();
-	const candidates = "candidate" in comparison ? [comparison.candidate] : comparison.candidates;
-	validateOptions(evalSet, comparison.baseline, candidates, repetitions, seed);
+	const candidates = "candidate" in options ? [options.candidate] : options.candidates;
+	validateOptions(evalSet, options.baseline, candidates, repetitions, seed);
 
 	const random = createSeededRandom(seed);
 	const rows: EvalHarnessTableRow<TInput, TOutput>[] = [];
-	const harnesses = [comparison.baseline, ...candidates];
+	const harnesses = [options.baseline, ...candidates];
 	let plannedOrder = 1;
 	for (let repetition = 1; repetition <= repetitions; repetition += 1) {
 		const randomizedHarnesses = [...harnesses];
@@ -216,7 +212,7 @@ export function evalHarnessTable<TInput, TOutput extends JsonValue | undefined>(
 				schemaVersion: 1,
 				evalSet,
 				harness: harness.name,
-				baseline: comparison.baseline.name,
+				baseline: options.baseline.name,
 				candidates: candidates.map(({ name }) => name),
 				repetition,
 				seed,

packages/evals/src/vitest-evals/summary.ts

@@ -370,8 +370,11 @@ function formatMetric(
 	if (metric.baselineMean === null || metric.candidateMean === null || metric.meanDelta === null) {
 		return formatReportLine(label, `${styleText("yellow", "unavailable")}${coverage}`);
 	}
-	const delta = colorDelta(metric.meanDelta, `delta ${formatDelta(metric.meanDelta)}`, false);
-	const values = styleText("gray", `(${formatValue(metric.candidateMean)}, ${formatValue(metric.baselineMean)})`);
+	const delta = colorDelta(metric.meanDelta, formatDelta(metric.meanDelta), false);
+	const values = styleText(
+		"gray",
+		`(candidate ${formatValue(metric.candidateMean)}, baseline ${formatValue(metric.baselineMean)})`,
+	);
 	return formatReportLine(label, `${delta} ${values}${coverage}`);
 }
 
@@ -394,10 +397,10 @@ export function formatHarnessComparisonReport(report: HarnessComparisonReport):
 				lines.push(formatReportLine("Pass rate", styleText("yellow", "unavailable")));
 			} else {
 				const lift = correctness.lift * 100;
-				const delta = colorDelta(lift, `delta ${formatSigned(lift, 1)} pp`, true);
+				const delta = colorDelta(lift, `${formatSigned(lift, 1)} pp`, true);
 				const values = styleText(
 					"gray",
-					`(${formatPercentage(correctness.candidatePassRate)}, ${formatPercentage(correctness.baselinePassRate)})`,
+					`(candidate ${formatPercentage(correctness.candidatePassRate)}, baseline ${formatPercentage(correctness.baselinePassRate)})`,
 				);
 				lines.push(formatReportLine("Pass rate", `${delta} ${values}`));
 			}

packages/evals/test/vitest-evals/harness-table.test.ts

@@ -43,14 +43,12 @@ function createFakeHarness(name: string) {
 	});
 }
 
-const harnessTable = evalHarnessTable(
-	"local multi-harness eval",
-	{
-		baseline: createFakeHarness("withoutSkill"),
-		candidates: [createFakeHarness("withSkill")],
-	},
-	{ repetitions: 2, seed: 42 },
-);
+const harnessTable = evalHarnessTable("local multi-harness eval", {
+	baseline: createFakeHarness("withoutSkill"),
+	candidates: [createFakeHarness("withSkill")],
+	repetitions: 2,
+	seed: 42,
+});
 
 describe("evalHarnessTable", () => {
 	it("plans seeded repetitions in deterministic order", () => {
@@ -65,11 +63,11 @@ describe("evalHarnessTable", () => {
 	});
 
 	it("accepts a singular candidate", () => {
-		const rows = evalHarnessTable(
-			"singular candidate",
-			{ baseline: createFakeHarness("baseline"), candidate: createFakeHarness("candidate") },
-			{ seed: 42 },
-		);
+		const rows = evalHarnessTable("singular candidate", {
+			baseline: createFakeHarness("baseline"),
+			candidate: createFakeHarness("candidate"),
+			seed: 42,
+		});
 
 		expect(rows.map(({ name }) => name)).toEqual(["baseline", "candidate"]);
 	});

packages/evals/test/vitest-evals/summary.test.ts

@@ -226,8 +226,8 @@ describe("summarizeHarnessComparisons", () => {
 		expect(formatted).toContain("Eval Comparisons");
 		expect(formatted).toContain(" Baseline  without-tools");
 		expect(formatted).toContain("Candidate  with-tools (1/1 pairs)");
-		expect(formatted).toContain("Pass rate  delta +100.0 pp (100.0%, 0.0%)");
+		expect(formatted).toContain("Pass rate  +100.0 pp (candidate 100.0%, baseline 0.0%)");
 		expect(formatted).toContain("   Tokens  unavailable");
-		expect(formatted).toContain("  Latency  delta -4159.5ms (30694.2ms, 34853.7ms)");
+		expect(formatted).toContain("  Latency  -4159.5ms (candidate 30694.2ms, baseline 34853.7ms)");
 	});
 });