feat(agent): benchmark session repository forks

packages/agent/benchmark/session/benchmark.ts

@@ -13,10 +13,10 @@ const READ_BENCHMARK_OPTIONS = {
 	warmupIterations: 5,
 } as const;
 
-export const WRITE_BENCHMARK_ITERATIONS = 30;
-export const WRITE_BENCHMARK_WARMUP_ITERATIONS = 5;
+const WRITE_BENCHMARK_ITERATIONS = 30;
+const WRITE_BENCHMARK_WARMUP_ITERATIONS = 5;
 
-export const WRITE_BENCHMARK_OPTIONS = {
+const WRITE_BENCHMARK_OPTIONS = {
 	time: 0,
 	iterations: WRITE_BENCHMARK_ITERATIONS,
 	warmupTime: 0,
@@ -51,6 +51,28 @@ interface PreparedReadFixture<TFixture> {
 	readonly fixture: TFixture;
 }
 
+interface NamedBenchmarkScenario {
+	readonly name: string;
+}
+
+interface RegisterWriteBenchmarksOptions<
+	TScenario extends NamedBenchmarkScenario,
+	TFixture extends AsyncDisposable,
+	TSubject,
+> {
+	readonly targets: readonly BenchmarkTarget<TFixture>[];
+	readonly scenarios: readonly TScenario[];
+	prepare(fixture: TFixture, scenario: TScenario): Promise<TSubject>;
+	expectedResult(scenario: TScenario): number;
+	run(subject: TSubject, scenario: TScenario): Promise<number>;
+}
+
+interface PreparedWriteFixtures<TSubject> {
+	readonly scenarioName: string;
+	readonly targetName: string;
+	readonly pendingSubjects: TSubject[];
+}
+
 /** Prepares, validates, registers, and disposes one immutable fixture per target and dataset. */
 export async function registerReadBenchmarks<
 	TDataset extends NamedBenchmarkDataset,
@@ -99,3 +121,59 @@ export async function registerReadBenchmarks<
 		}
 	}
 }
+
+/** Prepares, validates, registers, and disposes one equivalent fixture per write invocation. */
+export async function registerWriteBenchmarks<
+	TScenario extends NamedBenchmarkScenario,
+	TFixture extends AsyncDisposable,
+	TSubject,
+>(options: RegisterWriteBenchmarksOptions<TScenario, TFixture, TSubject>): Promise<void> {
+	const fixtures: TFixture[] = [];
+	const preparedFixtures: PreparedWriteFixtures<TSubject>[] = [];
+
+	process.once("beforeExit", async () => {
+		await Promise.all(fixtures.map((fixture) => fixture[Symbol.asyncDispose]()));
+	});
+
+	for (const scenario of options.scenarios) {
+		for (const target of options.targets) {
+			const validationFixture = await target.createFixture();
+			fixtures.push(validationFixture);
+			const validationSubject = await options.prepare(validationFixture, scenario);
+			strictEqual(await options.run(validationSubject, scenario), options.expectedResult(scenario));
+
+			const pendingSubjects: TSubject[] = [];
+			for (
+				let index = 0;
+				index < WRITE_BENCHMARK_WARMUP_ITERATIONS + WRITE_BENCHMARK_ITERATIONS;
+				index++
+			) {
+				const fixture = await target.createFixture();
+				fixtures.push(fixture);
+				pendingSubjects.push(await options.prepare(fixture, scenario));
+			}
+			preparedFixtures.push({ scenarioName: scenario.name, targetName: target.name, pendingSubjects });
+		}
+	}
+
+	for (const scenario of options.scenarios) {
+		describe(scenario.name, () => {
+			for (const target of options.targets) {
+				const prepared = preparedFixtures.find(
+					(candidate) => candidate.scenarioName === scenario.name && candidate.targetName === target.name,
+				);
+				if (prepared === undefined) throw new Error("Benchmark fixtures were not initialized");
+
+				bench(
+					target.name,
+					async () => {
+						const subject = prepared.pendingSubjects.shift();
+						if (subject === undefined) throw new Error("Benchmark fixture pool was exhausted");
+						await options.run(subject, scenario);
+					},
+					WRITE_BENCHMARK_OPTIONS,
+				);
+			}
+		});
+	}
+}

packages/agent/benchmark/session/session-repo.bench.ts

@@ -1,9 +1,12 @@
 import {
 	SESSION_REPO_CATALOG_BENCHMARK_DATASETS,
 	SESSION_REPO_CATALOG_READ_BENCHMARK_SCENARIOS,
+	SESSION_REPO_FORK_BENCHMARK_DATASETS,
+	SESSION_REPO_FORK_WRITE_BENCHMARK_SCENARIOS,
 	seedSessionRepoCatalogBenchmark,
+	seedSessionRepoForkBenchmark,
 } from "../../src/harness/session/testing/index.ts";
-import { registerReadBenchmarks } from "./benchmark.ts";
+import { registerReadBenchmarks, registerWriteBenchmarks } from "./benchmark.ts";
 import { sessionRepoBenchmarkTargets } from "./session-repo-targets.ts";
 
 await registerReadBenchmarks({
@@ -17,3 +20,28 @@ await registerReadBenchmarks({
 		return fixture.repo;
 	},
 });
+
+const forkBenchmarks = SESSION_REPO_FORK_BENCHMARK_DATASETS.flatMap((dataset) =>
+	SESSION_REPO_FORK_WRITE_BENCHMARK_SCENARIOS.map((scenario) => ({
+		name: `${scenario.name} (${dataset.name})`,
+		dataset,
+		scenario,
+	})),
+);
+
+await registerWriteBenchmarks({
+	targets: sessionRepoBenchmarkTargets,
+	scenarios: forkBenchmarks,
+	async prepare(fixture, benchmark) {
+		return {
+			repo: fixture.repo,
+			source: await seedSessionRepoForkBenchmark(fixture.repo, benchmark.dataset),
+		};
+	},
+	expectedResult(benchmark) {
+		return benchmark.scenario.expectedResult(benchmark.dataset);
+	},
+	run(subject, benchmark) {
+		return benchmark.scenario.run(subject.repo, subject.source, benchmark.dataset);
+	},
+});

packages/agent/benchmark/session/storage.bench.ts

@@ -1,33 +1,12 @@
-import { strictEqual } from "node:assert/strict";
-import { bench, describe } from "vitest";
 import {
 	STORAGE_BENCHMARK_DATASETS,
 	STORAGE_READ_BENCHMARK_SCENARIOS,
 	STORAGE_WRITE_BENCHMARK_SCENARIOS,
 	seedStorageBenchmark,
-	type StorageFixture,
 } from "../../src/harness/session/testing/index.ts";
-import {
-	registerReadBenchmarks,
-	WRITE_BENCHMARK_ITERATIONS,
-	WRITE_BENCHMARK_OPTIONS,
-	WRITE_BENCHMARK_WARMUP_ITERATIONS,
-} from "./benchmark.ts";
+import { registerReadBenchmarks, registerWriteBenchmarks } from "./benchmark.ts";
 import { storageBenchmarkTargets } from "./storage-targets.ts";
 
-interface PreparedWriteFixtures {
-	readonly scenarioName: string;
-	readonly targetName: string;
-	readonly pendingFixtures: StorageFixture[];
-}
-
-const allFixtures: StorageFixture[] = [];
-const writeFixtures: PreparedWriteFixtures[] = [];
-
-process.once("beforeExit", async () => {
-	await Promise.all(allFixtures.map((fixture) => fixture[Symbol.asyncDispose]()));
-});
-
 await registerReadBenchmarks({
 	datasets: STORAGE_BENCHMARK_DATASETS,
 	targets: storageBenchmarkTargets,
@@ -40,48 +19,17 @@ await registerReadBenchmarks({
 	},
 });
 
-// Vitest 4's benchmark runner does not execute normal test lifecycle hooks.
-// Prepare one write fixture per warmup/measured iteration before registration.
-// Vitest 5 can express the same lifecycle with each registration's hooks.
-for (const scenario of STORAGE_WRITE_BENCHMARK_SCENARIOS) {
-	for (const target of storageBenchmarkTargets) {
-		const validationFixture = await target.createFixture();
-		allFixtures.push(validationFixture);
-		await scenario.prepare?.(validationFixture.storage);
-		strictEqual(await scenario.run(validationFixture.storage), scenario.writeCount);
-
-		const pendingFixtures: StorageFixture[] = [];
-		for (
-			let index = 0;
-			index < WRITE_BENCHMARK_WARMUP_ITERATIONS + WRITE_BENCHMARK_ITERATIONS;
-			index++
-		) {
-			const fixture = await target.createFixture();
-			allFixtures.push(fixture);
-			await scenario.prepare?.(fixture.storage);
-			pendingFixtures.push(fixture);
-		}
-		writeFixtures.push({ scenarioName: scenario.name, targetName: target.name, pendingFixtures });
-	}
-}
-
-for (const scenario of STORAGE_WRITE_BENCHMARK_SCENARIOS) {
-	describe(scenario.name, () => {
-		for (const target of storageBenchmarkTargets) {
-			const prepared = writeFixtures.find(
-				(candidate) => candidate.scenarioName === scenario.name && candidate.targetName === target.name,
-			);
-			if (prepared === undefined) throw new Error("Benchmark fixtures were not initialized");
-
-			bench(
-				target.name,
-				async () => {
-					const fixture = prepared.pendingFixtures.shift();
-					if (fixture === undefined) throw new Error("Benchmark fixture pool was exhausted");
-					await scenario.run(fixture.storage);
-				},
-				WRITE_BENCHMARK_OPTIONS,
-			);
-		}
-	});
-}
+await registerWriteBenchmarks({
+	targets: storageBenchmarkTargets,
+	scenarios: STORAGE_WRITE_BENCHMARK_SCENARIOS,
+	async prepare(fixture, scenario) {
+		await scenario.prepare?.(fixture.storage);
+		return fixture.storage;
+	},
+	expectedResult(scenario) {
+		return scenario.writeCount;
+	},
+	run(storage, scenario) {
+		return scenario.run(storage);
+	},
+});

packages/agent/src/harness/session/testing/benchmark/session-repo.ts

@@ -1,4 +1,6 @@
 import type { SessionMetadata, SessionRepo } from "../../types.ts";
+import { STORAGE_BENCHMARK_DATASETS, type StorageBenchmarkDataset } from "./datasets.ts";
+import { generateStorageBenchmarkSeedTransactions } from "./storage.ts";
 
 export interface SessionRepoCatalogBenchmarkDataset {
 	readonly name: string;
@@ -11,11 +13,20 @@ interface SessionRepoCatalogReadBenchmarkScenario {
 	run(repo: SessionRepo): Promise<number>;
 }
 
+interface SessionRepoForkWriteBenchmarkScenario {
+	readonly name: string;
+	expectedResult(dataset: StorageBenchmarkDataset): number;
+	run(repo: SessionRepo, source: SessionMetadata, dataset: StorageBenchmarkDataset): Promise<number>;
+}
+
 /** Package-internal deterministic session id shared by repository benchmark workloads. */
 export function sessionRepoBenchmarkSessionId(index: number): string {
 	return `benchmark-session-${index.toString().padStart(8, "0")}`;
 }
 
+const FORK_SOURCE_SESSION_ID = sessionRepoBenchmarkSessionId(0);
+const FORK_DESTINATION_SESSION_ID = sessionRepoBenchmarkSessionId(1);
+
 function createCatalogDataset(scale: string, sessionCount: number): SessionRepoCatalogBenchmarkDataset {
 	return {
 		name: `synthetic catalog: ${scale} closed sessions`,
@@ -56,3 +67,42 @@ export const SESSION_REPO_CATALOG_READ_BENCHMARK_SCENARIOS: readonly SessionRepo
 		},
 	},
 ];
+
+/** Initial fork timing uses a bounded source because each iteration owns an equivalent seeded repository. */
+export const SESSION_REPO_FORK_BENCHMARK_DATASETS: readonly StorageBenchmarkDataset[] = [
+	STORAGE_BENCHMARK_DATASETS[0]!,
+	STORAGE_BENCHMARK_DATASETS[1]!,
+];
+
+/** Seeds one deterministic open source session with a linear main branch. */
+export async function seedSessionRepoForkBenchmark(
+	repo: SessionRepo,
+	dataset: StorageBenchmarkDataset,
+): Promise<SessionMetadata> {
+	const session = await repo.create({ id: FORK_SOURCE_SESSION_ID });
+	for (const transaction of generateStorageBenchmarkSeedTransactions(dataset)) {
+		await session.mutate("main", (mutator) => mutator.commit(transaction));
+	}
+	await session.mutate("main", (mutator) =>
+		mutator.commit({
+			writes: [{ kind: "register", op: "set", namespace: "lane.leaf", key: "main", value: dataset.leafId }],
+		}),
+	);
+	return session.metadata;
+}
+
+/** Shared fork writes. Every invocation receives an equivalent source repository. */
+export const SESSION_REPO_FORK_WRITE_BENCHMARK_SCENARIOS: readonly SessionRepoForkWriteBenchmarkScenario[] = [
+	{
+		name: "fork open current branch",
+		expectedResult(dataset) {
+			return dataset.entryCount;
+		},
+		async run(repo, source, dataset) {
+			const fork = await repo.fork(source, { id: FORK_DESTINATION_SESSION_ID });
+			return fork.metadata.id === FORK_DESTINATION_SESSION_ID && fork.metadata.parentSessionId === source.id
+				? dataset.entryCount
+				: 0;
+		},
+	},
+];

packages/agent/src/harness/session/testing/index.ts

@@ -2,8 +2,11 @@ export { STORAGE_BENCHMARK_DATASETS } from "./benchmark/datasets.ts";
 export {
 	SESSION_REPO_CATALOG_BENCHMARK_DATASETS,
 	SESSION_REPO_CATALOG_READ_BENCHMARK_SCENARIOS,
+	SESSION_REPO_FORK_BENCHMARK_DATASETS,
+	SESSION_REPO_FORK_WRITE_BENCHMARK_SCENARIOS,
 	type SessionRepoCatalogBenchmarkDataset,
 	seedSessionRepoCatalogBenchmark,
+	seedSessionRepoForkBenchmark,
 	sessionRepoBenchmarkSessionId,
 } from "./benchmark/session-repo.ts";
 export {