diff --git a/packages/agent/benchmark/session/benchmark.ts b/packages/agent/benchmark/session/benchmark.ts index 6dbb72a99..8abd56ec5 100644 --- a/packages/agent/benchmark/session/benchmark.ts +++ b/packages/agent/benchmark/session/benchmark.ts @@ -13,10 +13,10 @@ const READ_BENCHMARK_OPTIONS = { warmupIterations: 5, } as const; -export const WRITE_BENCHMARK_ITERATIONS = 30; -export const WRITE_BENCHMARK_WARMUP_ITERATIONS = 5; +const WRITE_BENCHMARK_ITERATIONS = 30; +const WRITE_BENCHMARK_WARMUP_ITERATIONS = 5; -export const WRITE_BENCHMARK_OPTIONS = { +const WRITE_BENCHMARK_OPTIONS = { time: 0, iterations: WRITE_BENCHMARK_ITERATIONS, warmupTime: 0, @@ -51,6 +51,28 @@ interface PreparedReadFixture { readonly fixture: TFixture; } +interface NamedBenchmarkScenario { + readonly name: string; +} + +interface RegisterWriteBenchmarksOptions< + TScenario extends NamedBenchmarkScenario, + TFixture extends AsyncDisposable, + TSubject, +> { + readonly targets: readonly BenchmarkTarget[]; + readonly scenarios: readonly TScenario[]; + prepare(fixture: TFixture, scenario: TScenario): Promise; + expectedResult(scenario: TScenario): number; + run(subject: TSubject, scenario: TScenario): Promise; +} + +interface PreparedWriteFixtures { + readonly scenarioName: string; + readonly targetName: string; + readonly pendingSubjects: TSubject[]; +} + /** Prepares, validates, registers, and disposes one immutable fixture per target and dataset. */ export async function registerReadBenchmarks< TDataset extends NamedBenchmarkDataset, @@ -99,3 +121,59 @@ export async function registerReadBenchmarks< } } } + +/** Prepares, validates, registers, and disposes one equivalent fixture per write invocation. */ +export async function registerWriteBenchmarks< + TScenario extends NamedBenchmarkScenario, + TFixture extends AsyncDisposable, + TSubject, +>(options: RegisterWriteBenchmarksOptions): Promise { + const fixtures: TFixture[] = []; + const preparedFixtures: PreparedWriteFixtures[] = []; + + process.once("beforeExit", async () => { + await Promise.all(fixtures.map((fixture) => fixture[Symbol.asyncDispose]())); + }); + + for (const scenario of options.scenarios) { + for (const target of options.targets) { + const validationFixture = await target.createFixture(); + fixtures.push(validationFixture); + const validationSubject = await options.prepare(validationFixture, scenario); + strictEqual(await options.run(validationSubject, scenario), options.expectedResult(scenario)); + + const pendingSubjects: TSubject[] = []; + for ( + let index = 0; + index < WRITE_BENCHMARK_WARMUP_ITERATIONS + WRITE_BENCHMARK_ITERATIONS; + index++ + ) { + const fixture = await target.createFixture(); + fixtures.push(fixture); + pendingSubjects.push(await options.prepare(fixture, scenario)); + } + preparedFixtures.push({ scenarioName: scenario.name, targetName: target.name, pendingSubjects }); + } + } + + for (const scenario of options.scenarios) { + describe(scenario.name, () => { + for (const target of options.targets) { + const prepared = preparedFixtures.find( + (candidate) => candidate.scenarioName === scenario.name && candidate.targetName === target.name, + ); + if (prepared === undefined) throw new Error("Benchmark fixtures were not initialized"); + + bench( + target.name, + async () => { + const subject = prepared.pendingSubjects.shift(); + if (subject === undefined) throw new Error("Benchmark fixture pool was exhausted"); + await options.run(subject, scenario); + }, + WRITE_BENCHMARK_OPTIONS, + ); + } + }); + } +} diff --git a/packages/agent/benchmark/session/session-repo.bench.ts b/packages/agent/benchmark/session/session-repo.bench.ts index 0498f0aa5..c8fc65e89 100644 --- a/packages/agent/benchmark/session/session-repo.bench.ts +++ b/packages/agent/benchmark/session/session-repo.bench.ts @@ -1,9 +1,12 @@ import { SESSION_REPO_CATALOG_BENCHMARK_DATASETS, SESSION_REPO_CATALOG_READ_BENCHMARK_SCENARIOS, + SESSION_REPO_FORK_BENCHMARK_DATASETS, + SESSION_REPO_FORK_WRITE_BENCHMARK_SCENARIOS, seedSessionRepoCatalogBenchmark, + seedSessionRepoForkBenchmark, } from "../../src/harness/session/testing/index.ts"; -import { registerReadBenchmarks } from "./benchmark.ts"; +import { registerReadBenchmarks, registerWriteBenchmarks } from "./benchmark.ts"; import { sessionRepoBenchmarkTargets } from "./session-repo-targets.ts"; await registerReadBenchmarks({ @@ -17,3 +20,28 @@ await registerReadBenchmarks({ return fixture.repo; }, }); + +const forkBenchmarks = SESSION_REPO_FORK_BENCHMARK_DATASETS.flatMap((dataset) => + SESSION_REPO_FORK_WRITE_BENCHMARK_SCENARIOS.map((scenario) => ({ + name: `${scenario.name} (${dataset.name})`, + dataset, + scenario, + })), +); + +await registerWriteBenchmarks({ + targets: sessionRepoBenchmarkTargets, + scenarios: forkBenchmarks, + async prepare(fixture, benchmark) { + return { + repo: fixture.repo, + source: await seedSessionRepoForkBenchmark(fixture.repo, benchmark.dataset), + }; + }, + expectedResult(benchmark) { + return benchmark.scenario.expectedResult(benchmark.dataset); + }, + run(subject, benchmark) { + return benchmark.scenario.run(subject.repo, subject.source, benchmark.dataset); + }, +}); diff --git a/packages/agent/benchmark/session/storage.bench.ts b/packages/agent/benchmark/session/storage.bench.ts index b0f5823d5..ca35b120c 100644 --- a/packages/agent/benchmark/session/storage.bench.ts +++ b/packages/agent/benchmark/session/storage.bench.ts @@ -1,33 +1,12 @@ -import { strictEqual } from "node:assert/strict"; -import { bench, describe } from "vitest"; import { STORAGE_BENCHMARK_DATASETS, STORAGE_READ_BENCHMARK_SCENARIOS, STORAGE_WRITE_BENCHMARK_SCENARIOS, seedStorageBenchmark, - type StorageFixture, } from "../../src/harness/session/testing/index.ts"; -import { - registerReadBenchmarks, - WRITE_BENCHMARK_ITERATIONS, - WRITE_BENCHMARK_OPTIONS, - WRITE_BENCHMARK_WARMUP_ITERATIONS, -} from "./benchmark.ts"; +import { registerReadBenchmarks, registerWriteBenchmarks } from "./benchmark.ts"; import { storageBenchmarkTargets } from "./storage-targets.ts"; -interface PreparedWriteFixtures { - readonly scenarioName: string; - readonly targetName: string; - readonly pendingFixtures: StorageFixture[]; -} - -const allFixtures: StorageFixture[] = []; -const writeFixtures: PreparedWriteFixtures[] = []; - -process.once("beforeExit", async () => { - await Promise.all(allFixtures.map((fixture) => fixture[Symbol.asyncDispose]())); -}); - await registerReadBenchmarks({ datasets: STORAGE_BENCHMARK_DATASETS, targets: storageBenchmarkTargets, @@ -40,48 +19,17 @@ await registerReadBenchmarks({ }, }); -// Vitest 4's benchmark runner does not execute normal test lifecycle hooks. -// Prepare one write fixture per warmup/measured iteration before registration. -// Vitest 5 can express the same lifecycle with each registration's hooks. -for (const scenario of STORAGE_WRITE_BENCHMARK_SCENARIOS) { - for (const target of storageBenchmarkTargets) { - const validationFixture = await target.createFixture(); - allFixtures.push(validationFixture); - await scenario.prepare?.(validationFixture.storage); - strictEqual(await scenario.run(validationFixture.storage), scenario.writeCount); - - const pendingFixtures: StorageFixture[] = []; - for ( - let index = 0; - index < WRITE_BENCHMARK_WARMUP_ITERATIONS + WRITE_BENCHMARK_ITERATIONS; - index++ - ) { - const fixture = await target.createFixture(); - allFixtures.push(fixture); - await scenario.prepare?.(fixture.storage); - pendingFixtures.push(fixture); - } - writeFixtures.push({ scenarioName: scenario.name, targetName: target.name, pendingFixtures }); - } -} - -for (const scenario of STORAGE_WRITE_BENCHMARK_SCENARIOS) { - describe(scenario.name, () => { - for (const target of storageBenchmarkTargets) { - const prepared = writeFixtures.find( - (candidate) => candidate.scenarioName === scenario.name && candidate.targetName === target.name, - ); - if (prepared === undefined) throw new Error("Benchmark fixtures were not initialized"); - - bench( - target.name, - async () => { - const fixture = prepared.pendingFixtures.shift(); - if (fixture === undefined) throw new Error("Benchmark fixture pool was exhausted"); - await scenario.run(fixture.storage); - }, - WRITE_BENCHMARK_OPTIONS, - ); - } - }); -} +await registerWriteBenchmarks({ + targets: storageBenchmarkTargets, + scenarios: STORAGE_WRITE_BENCHMARK_SCENARIOS, + async prepare(fixture, scenario) { + await scenario.prepare?.(fixture.storage); + return fixture.storage; + }, + expectedResult(scenario) { + return scenario.writeCount; + }, + run(storage, scenario) { + return scenario.run(storage); + }, +}); diff --git a/packages/agent/src/harness/session/testing/benchmark/session-repo.ts b/packages/agent/src/harness/session/testing/benchmark/session-repo.ts index 9b6afb62f..e495b3c96 100644 --- a/packages/agent/src/harness/session/testing/benchmark/session-repo.ts +++ b/packages/agent/src/harness/session/testing/benchmark/session-repo.ts @@ -1,4 +1,6 @@ import type { SessionMetadata, SessionRepo } from "../../types.ts"; +import { STORAGE_BENCHMARK_DATASETS, type StorageBenchmarkDataset } from "./datasets.ts"; +import { generateStorageBenchmarkSeedTransactions } from "./storage.ts"; export interface SessionRepoCatalogBenchmarkDataset { readonly name: string; @@ -11,11 +13,20 @@ interface SessionRepoCatalogReadBenchmarkScenario { run(repo: SessionRepo): Promise; } +interface SessionRepoForkWriteBenchmarkScenario { + readonly name: string; + expectedResult(dataset: StorageBenchmarkDataset): number; + run(repo: SessionRepo, source: SessionMetadata, dataset: StorageBenchmarkDataset): Promise; +} + /** Package-internal deterministic session id shared by repository benchmark workloads. */ export function sessionRepoBenchmarkSessionId(index: number): string { return `benchmark-session-${index.toString().padStart(8, "0")}`; } +const FORK_SOURCE_SESSION_ID = sessionRepoBenchmarkSessionId(0); +const FORK_DESTINATION_SESSION_ID = sessionRepoBenchmarkSessionId(1); + function createCatalogDataset(scale: string, sessionCount: number): SessionRepoCatalogBenchmarkDataset { return { name: `synthetic catalog: ${scale} closed sessions`, @@ -56,3 +67,42 @@ export const SESSION_REPO_CATALOG_READ_BENCHMARK_SCENARIOS: readonly SessionRepo }, }, ]; + +/** Initial fork timing uses a bounded source because each iteration owns an equivalent seeded repository. */ +export const SESSION_REPO_FORK_BENCHMARK_DATASETS: readonly StorageBenchmarkDataset[] = [ + STORAGE_BENCHMARK_DATASETS[0]!, + STORAGE_BENCHMARK_DATASETS[1]!, +]; + +/** Seeds one deterministic open source session with a linear main branch. */ +export async function seedSessionRepoForkBenchmark( + repo: SessionRepo, + dataset: StorageBenchmarkDataset, +): Promise { + const session = await repo.create({ id: FORK_SOURCE_SESSION_ID }); + for (const transaction of generateStorageBenchmarkSeedTransactions(dataset)) { + await session.mutate("main", (mutator) => mutator.commit(transaction)); + } + await session.mutate("main", (mutator) => + mutator.commit({ + writes: [{ kind: "register", op: "set", namespace: "lane.leaf", key: "main", value: dataset.leafId }], + }), + ); + return session.metadata; +} + +/** Shared fork writes. Every invocation receives an equivalent source repository. */ +export const SESSION_REPO_FORK_WRITE_BENCHMARK_SCENARIOS: readonly SessionRepoForkWriteBenchmarkScenario[] = [ + { + name: "fork open current branch", + expectedResult(dataset) { + return dataset.entryCount; + }, + async run(repo, source, dataset) { + const fork = await repo.fork(source, { id: FORK_DESTINATION_SESSION_ID }); + return fork.metadata.id === FORK_DESTINATION_SESSION_ID && fork.metadata.parentSessionId === source.id + ? dataset.entryCount + : 0; + }, + }, +]; diff --git a/packages/agent/src/harness/session/testing/index.ts b/packages/agent/src/harness/session/testing/index.ts index 111926263..28143a871 100644 --- a/packages/agent/src/harness/session/testing/index.ts +++ b/packages/agent/src/harness/session/testing/index.ts @@ -2,8 +2,11 @@ export { STORAGE_BENCHMARK_DATASETS } from "./benchmark/datasets.ts"; export { SESSION_REPO_CATALOG_BENCHMARK_DATASETS, SESSION_REPO_CATALOG_READ_BENCHMARK_SCENARIOS, + SESSION_REPO_FORK_BENCHMARK_DATASETS, + SESSION_REPO_FORK_WRITE_BENCHMARK_SCENARIOS, type SessionRepoCatalogBenchmarkDataset, seedSessionRepoCatalogBenchmark, + seedSessionRepoForkBenchmark, sessionRepoBenchmarkSessionId, } from "./benchmark/session-repo.ts"; export {