feat(agent): add session storage benchmarks

Add reusable timing, loaded-footprint, and allocation workloads and runners. Index memory entries by sequence so limited scans can stop early.

packages/agent/benchmark/session/README.md

@@ -0,0 +1,34 @@
+# Session storage benchmarks
+
+Run these commands from `packages/agent`.
+
+These benchmarks use deterministic synthetic data. The scale datasets are linear branches of user-message entries with fixed 256-byte text payloads, deterministic IDs, and 250-entry seed transactions. They exercise explicit storage structures and make no claim to reproduce production traffic.
+
+## Timing
+
+The timing suite runs steady-state scenarios against every target in `storage-targets.ts`. Read scenarios reuse one immutable fixture per backend/dataset and validate their result once before timing. Every warmup and measured write runs once against an independently prepared fixture, so entry counts, sequence numbers, and durable artifacts start in equivalent states. Implementations are registered under the same Vitest suite so additional backends are directly comparable.
+
+Initial write scenarios cover a single message, a 100-message transaction, and a mixed message/register/usage append to a 1k-entry synthetic branch. Fixture preparation, transaction generation, and validation happen outside the measured callback.
+
+```sh
+npm run bench:session:timing
+npm run bench:session:timing -- -t "scan latest"
+```
+
+## Loaded footprint
+
+The memory profile starts a fresh Node.js process with forced GC for every backend/dataset pair. Its baseline is recorded before generating and ingesting the dataset, so the result includes retained payload data and backend structures. Temporary generation data should be collected before the final reading, while RSS can still reflect allocator growth.
+
+```sh
+npm run bench:session:memory
+```
+
+## Storage allocation sampling
+
+The allocation profile prebuilds all transactions, then samples allocations made while committing them. This excludes synthetic payload generation and reports estimated cumulative allocated bytes, including objects later collected, plus the largest sampled allocation sites. V8 sampling is approximate and does not report an exact object count.
+
+```sh
+npm run bench:session:allocations
+```
+
+Timing results are not CI performance gates. Run them on an otherwise idle machine and compare results produced on the same hardware and Node.js version.

packages/agent/benchmark/session/allocation-profile-worker.ts

@@ -0,0 +1,57 @@
+import type { HeapProfiler } from "node:inspector";
+import { Session } from "node:inspector/promises";
+import {
+	STORAGE_BENCHMARK_DATASETS,
+	generateStorageBenchmarkSeedTransactions,
+} from "../../src/harness/session/testing/index.ts";
+import { storageBenchmarkTargets } from "./storage-targets.ts";
+
+const SAMPLING_INTERVAL_BYTES = 4_096;
+const TOP_SITE_COUNT = 5;
+
+function collectAllocationSites(node: HeapProfiler.SamplingHeapProfileNode, sites: Map<string, number>): number {
+	const { functionName, url, lineNumber, columnNumber } = node.callFrame;
+	const site = `${functionName || "(anonymous)"} (${url || "native"}:${lineNumber + 1}:${columnNumber + 1})`;
+	if (node.selfSize > 0) sites.set(site, (sites.get(site) ?? 0) + node.selfSize);
+	let total = node.selfSize;
+	for (const child of node.children) total += collectAllocationSites(child, sites);
+	return total;
+}
+
+const [targetName, datasetName] = process.argv.slice(2);
+const target = storageBenchmarkTargets.find((candidate) => candidate.name === targetName);
+if (target === undefined) throw new Error(`Unknown storage benchmark target: ${targetName}`);
+const dataset = STORAGE_BENCHMARK_DATASETS.find((candidate) => candidate.name === datasetName);
+if (dataset === undefined) throw new Error(`Unknown storage benchmark dataset: ${datasetName}`);
+
+// Inputs and the empty fixture exist before sampling so the profile measures
+// allocation caused by Storage.commit(), not synthetic payload generation.
+const transactions = [...generateStorageBenchmarkSeedTransactions(dataset)];
+await using fixture = await target.createFixture();
+const inspector = new Session();
+inspector.connect();
+try {
+	await inspector.post("HeapProfiler.startSampling", {
+		samplingInterval: SAMPLING_INTERVAL_BYTES,
+		includeObjectsCollectedByMajorGC: true,
+		includeObjectsCollectedByMinorGC: true,
+	});
+	for (const transaction of transactions) await fixture.storage.commit(transaction);
+	const { profile } = await inspector.post("HeapProfiler.stopSampling");
+	const sites = new Map<string, number>();
+	const allocatedBytes = collectAllocationSites(profile.head, sites);
+	console.log(
+		JSON.stringify({
+			target: target.name,
+			dataset: dataset.name,
+			entryCount: dataset.entryCount,
+			allocatedBytes,
+			topSites: [...sites.entries()]
+				.map(([site, bytes]) => ({ site, bytes }))
+				.sort((left, right) => right.bytes - left.bytes)
+				.slice(0, TOP_SITE_COUNT),
+		}),
+	);
+} finally {
+	inspector.disconnect();
+}

packages/agent/benchmark/session/allocation-profile.ts

@@ -0,0 +1,55 @@
+import { execFile } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+import { STORAGE_BENCHMARK_DATASETS } from "../../src/harness/session/testing/index.ts";
+import { storageBenchmarkTargets } from "./storage-targets.ts";
+
+interface AllocationSiteResult {
+	readonly site: string;
+	readonly bytes: number;
+}
+
+interface AllocationProfileResult {
+	readonly target: string;
+	readonly dataset: string;
+	readonly entryCount: number;
+	readonly allocatedBytes: number;
+	readonly topSites: readonly AllocationSiteResult[];
+}
+
+const execFileAsync = promisify(execFile);
+const packageRoot = fileURLToPath(new URL("../..", import.meta.url));
+const workerPath = fileURLToPath(new URL("./allocation-profile-worker.ts", import.meta.url));
+const tsconfigPath = fileURLToPath(new URL("../tsconfig.json", import.meta.url));
+const results: AllocationProfileResult[] = [];
+
+for (const target of storageBenchmarkTargets) {
+	for (const dataset of STORAGE_BENCHMARK_DATASETS) {
+		const { stdout } = await execFileAsync(
+			process.execPath,
+			["--import", "tsx", workerPath, target.name, dataset.name],
+			{ cwd: packageRoot, env: { ...process.env, TSX_TSCONFIG_PATH: tsconfigPath } },
+		);
+		results.push(JSON.parse(stdout) as AllocationProfileResult);
+	}
+}
+
+const mebibytes = (bytes: number): string => (bytes / 1024 / 1024).toFixed(2);
+console.log("Storage allocation sampling (inputs generated before sampling):");
+console.table(
+	results.map((result) => ({
+		backend: result.target,
+		dataset: result.dataset,
+		"allocated MiB": mebibytes(result.allocatedBytes),
+		"allocated bytes/entry": Math.round(result.allocatedBytes / result.entryCount),
+	})),
+);
+for (const result of results) {
+	console.log(`\n${result.target}, ${result.dataset}, top sampled allocation sites:`);
+	console.table(
+		result.topSites.map(({ site, bytes }) => ({
+			site,
+			"sampled MiB": mebibytes(bytes),
+		})),
+	);
+}

packages/agent/benchmark/session/loaded-footprint-worker.ts

@@ -0,0 +1,34 @@
+import {
+	STORAGE_BENCHMARK_DATASETS,
+	seedStorageBenchmark,
+} from "../../src/harness/session/testing/index.ts";
+import { storageBenchmarkTargets } from "./storage-targets.ts";
+
+function collectGarbage(): void {
+	if (globalThis.gc === undefined) throw new Error("Loaded footprint measurement requires Node.js --expose-gc");
+	for (let index = 0; index < 3; index++) globalThis.gc();
+}
+
+const [targetName, datasetName] = process.argv.slice(2);
+const target = storageBenchmarkTargets.find((candidate) => candidate.name === targetName);
+if (target === undefined) throw new Error(`Unknown storage benchmark target: ${targetName}`);
+const dataset = STORAGE_BENCHMARK_DATASETS.find((candidate) => candidate.name === datasetName);
+if (dataset === undefined) throw new Error(`Unknown storage benchmark dataset: ${datasetName}`);
+
+await using fixture = await target.createFixture();
+collectGarbage();
+const before = process.memoryUsage();
+await seedStorageBenchmark(fixture.storage, dataset);
+collectGarbage();
+const after = process.memoryUsage();
+
+console.log(
+	JSON.stringify({
+		target: target.name,
+		dataset: dataset.name,
+		entryCount: dataset.entryCount,
+		heapUsedBytes: after.heapUsed - before.heapUsed,
+		rssBytes: after.rss - before.rss,
+		externalBytes: after.external - before.external,
+	}),
+);

packages/agent/benchmark/session/loaded-footprint.ts

@@ -0,0 +1,51 @@
+import { execFile } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
+import { STORAGE_BENCHMARK_DATASETS } from "../../src/harness/session/testing/index.ts";
+import { storageBenchmarkTargets } from "./storage-targets.ts";
+
+interface LoadedFootprintResult {
+	readonly target: string;
+	readonly dataset: string;
+	readonly entryCount: number;
+	readonly heapUsedBytes: number;
+	readonly rssBytes: number;
+	readonly externalBytes: number;
+}
+
+const execFileAsync = promisify(execFile);
+const packageRoot = fileURLToPath(new URL("../..", import.meta.url));
+const workerPath = fileURLToPath(new URL("./loaded-footprint-worker.ts", import.meta.url));
+const tsconfigPath = fileURLToPath(new URL("../tsconfig.json", import.meta.url));
+const results: LoadedFootprintResult[] = [];
+
+for (const target of storageBenchmarkTargets) {
+	for (const dataset of STORAGE_BENCHMARK_DATASETS) {
+		const { stdout } = await execFileAsync(
+			process.execPath,
+			[
+				"--expose-gc",
+				"--import",
+				"tsx",
+				workerPath,
+				target.name,
+				dataset.name,
+			],
+			{ cwd: packageRoot, env: { ...process.env, TSX_TSCONFIG_PATH: tsconfigPath } },
+		);
+		results.push(JSON.parse(stdout) as LoadedFootprintResult);
+	}
+}
+
+const mebibytes = (bytes: number): string => (bytes / 1024 / 1024).toFixed(2);
+console.log("Loaded footprint (synthetic payloads generated and ingested after the baseline):");
+console.table(
+	results.map((result) => ({
+		backend: result.target,
+		dataset: result.dataset,
+		"heap MiB": mebibytes(result.heapUsedBytes),
+		"heap bytes/entry": Math.round(result.heapUsedBytes / result.entryCount),
+		"RSS MiB": mebibytes(result.rssBytes),
+		"external MiB": mebibytes(result.externalBytes),
+	})),
+);

packages/agent/benchmark/session/storage-targets.ts

@@ -0,0 +1,22 @@
+import { MemoryStorage } from "../../src/harness/session/memory.ts";
+import type { StorageFixture } from "../../src/harness/session/testing/index.ts";
+
+interface StorageBenchmarkTarget {
+	readonly name: string;
+	createFixture(): Promise<StorageFixture>;
+}
+
+const NOW = 1_700_000_000_000;
+
+export const storageBenchmarkTargets = [
+	{
+		name: "memory",
+		createFixture() {
+			const storage = new MemoryStorage({ now: () => NOW });
+			return Promise.resolve({
+				storage,
+				[Symbol.asyncDispose]: () => storage.close(),
+			});
+		},
+	},
+] satisfies readonly StorageBenchmarkTarget[];

packages/agent/benchmark/session/storage.bench.ts

@@ -0,0 +1,124 @@
+import { strictEqual } from "node:assert/strict";
+import { bench, describe } from "vitest";
+import {
+	STORAGE_BENCHMARK_DATASETS,
+	STORAGE_READ_BENCHMARK_SCENARIOS,
+	STORAGE_WRITE_BENCHMARK_SCENARIOS,
+	seedStorageBenchmark,
+	type StorageFixture,
+} from "../../src/harness/session/testing/index.ts";
+import { storageBenchmarkTargets } from "./storage-targets.ts";
+
+const BENCHMARK_TIME_MS = 500;
+const BENCHMARK_WARMUP_TIME_MS = 100;
+const WRITE_ITERATIONS = 30;
+const WRITE_WARMUP_ITERATIONS = 5;
+
+interface PreparedReadFixture {
+	readonly datasetName: string;
+	readonly targetName: string;
+	readonly fixture: StorageFixture;
+}
+
+interface PreparedWriteFixtures {
+	readonly scenarioName: string;
+	readonly targetName: string;
+	readonly pendingFixtures: StorageFixture[];
+}
+
+let benchmarkSink = 0;
+const allFixtures: StorageFixture[] = [];
+const readFixtures: PreparedReadFixture[] = [];
+const writeFixtures: PreparedWriteFixtures[] = [];
+
+process.once("beforeExit", () => {
+	void Promise.all(allFixtures.map((fixture) => fixture[Symbol.asyncDispose]()));
+});
+
+// Vitest 4's benchmark runner does not execute normal test lifecycle hooks.
+// Prepare read fixtures and one write fixture per warmup/measured iteration
+// before registration. Vitest 5 can express the same lifecycle with each
+// registration's beforeAll/beforeEach/afterEach/afterAll hooks.
+for (const dataset of STORAGE_BENCHMARK_DATASETS) {
+	for (const target of storageBenchmarkTargets) {
+		const fixture = await target.createFixture();
+		allFixtures.push(fixture);
+		await seedStorageBenchmark(fixture.storage, dataset);
+		for (const scenario of STORAGE_READ_BENCHMARK_SCENARIOS) {
+			strictEqual(await scenario.run(fixture.storage, dataset), scenario.expectedResult(dataset));
+		}
+		readFixtures.push({ datasetName: dataset.name, targetName: target.name, fixture });
+	}
+}
+
+for (const scenario of STORAGE_WRITE_BENCHMARK_SCENARIOS) {
+	for (const target of storageBenchmarkTargets) {
+		const validationFixture = await target.createFixture();
+		allFixtures.push(validationFixture);
+		await scenario.prepare?.(validationFixture.storage);
+		strictEqual(await scenario.run(validationFixture.storage), scenario.writeCount);
+
+		const pendingFixtures: StorageFixture[] = [];
+		for (let index = 0; index < WRITE_WARMUP_ITERATIONS + WRITE_ITERATIONS; index++) {
+			const fixture = await target.createFixture();
+			allFixtures.push(fixture);
+			await scenario.prepare?.(fixture.storage);
+			pendingFixtures.push(fixture);
+		}
+		writeFixtures.push({ scenarioName: scenario.name, targetName: target.name, pendingFixtures });
+	}
+}
+
+for (const dataset of STORAGE_BENCHMARK_DATASETS) {
+	for (const scenario of STORAGE_READ_BENCHMARK_SCENARIOS) {
+		describe(`${scenario.name} (${dataset.name})`, () => {
+			for (const target of storageBenchmarkTargets) {
+				const prepared = readFixtures.find(
+					(candidate) => candidate.datasetName === dataset.name && candidate.targetName === target.name,
+				);
+				if (prepared === undefined) throw new Error("Benchmark fixture was not initialized");
+
+				bench(
+					target.name,
+					async () => {
+						benchmarkSink = await scenario.run(prepared.fixture.storage, dataset);
+					},
+					{
+						time: BENCHMARK_TIME_MS,
+						iterations: 10,
+						warmupTime: BENCHMARK_WARMUP_TIME_MS,
+						warmupIterations: 5,
+					},
+				);
+			}
+		});
+	}
+}
+
+for (const scenario of STORAGE_WRITE_BENCHMARK_SCENARIOS) {
+	describe(scenario.name, () => {
+		for (const target of storageBenchmarkTargets) {
+			const prepared = writeFixtures.find(
+				(candidate) => candidate.scenarioName === scenario.name && candidate.targetName === target.name,
+			);
+			if (prepared === undefined) throw new Error("Benchmark fixtures were not initialized");
+
+			bench(
+				target.name,
+				async () => {
+					const fixture = prepared.pendingFixtures.shift();
+					if (fixture === undefined) throw new Error("Benchmark fixture pool was exhausted");
+					benchmarkSink = await scenario.run(fixture.storage);
+				},
+				{
+					time: 0,
+					iterations: WRITE_ITERATIONS,
+					warmupTime: 0,
+					warmupIterations: WRITE_WARMUP_ITERATIONS,
+				},
+			);
+		}
+	});
+}
+
+void benchmarkSink;

packages/agent/benchmark/tsconfig.json

@@ -0,0 +1,12 @@
+{
+	"extends": "../../../tsconfig.base.json",
+	"compilerOptions": {
+		"baseUrl": "../../..",
+		"noEmit": true,
+		"paths": {
+			"@earendil-works/pi-ai": ["packages/ai/src/index.ts"],
+			"@earendil-works/pi-ai/*": ["packages/ai/src/*"]
+		}
+	},
+	"include": ["./**/*.ts", "../src/**/*.ts"]
+}

packages/agent/package.json

@@ -29,8 +29,12 @@
 		"generate-telemetry-docs": "node scripts/generate-telemetry-docs.ts",
 		"check:telemetry-docs": "node scripts/generate-telemetry-docs.ts --check",
 		"build": "tsgo -p tsconfig.build.json",
+		"bench:session:timing": "vitest bench --config vitest.benchmark.config.ts",
+		"bench:session:memory": "tsx --tsconfig benchmark/tsconfig.json benchmark/session/loaded-footprint.ts",
+		"bench:session:allocations": "tsx --tsconfig benchmark/tsconfig.json benchmark/session/allocation-profile.ts",
 		"test": "vitest --run",
 		"test:harness": "vitest --run --config vitest.harness.config.ts",
+		"test:session:conformance": "vitest --run --config vitest.harness.config.ts test/harness/memory-conformance.test.ts",
 		"coverage:harness": "vitest --run --config vitest.harness.config.ts --coverage",
 		"prepublishOnly": "npm run build"
 	},

packages/agent/src/harness/session/memory.ts

@@ -55,6 +55,7 @@ function emptyUsage(): Usage {
 export class MemoryStorage implements Storage {
 	private readonly now: () => number;
 	private entries = new Map<string, Entry>();
+	private entriesBySeq: Entry[] = [];
 	private registers = new Map<string, Register>();
 	private usage = new Map<string, UsageRow>();
 	private stats: SessionStats = { messageCount: 0, usage: emptyUsage() };
@@ -138,7 +139,10 @@ export class MemoryStorage implements Storage {
 			}
 		}
 
-		for (const entry of entries) this.entries.set(entry.id, entry);
+		for (const entry of entries) {
+			this.entries.set(entry.id, entry);
+			this.entriesBySeq.push(entry);
+		}
 		for (const [key, register] of registers) {
 			if (register === undefined) this.registers.delete(key);
 			else this.registers.set(key, register);
@@ -228,13 +232,23 @@ export class MemoryStorage implements Storage {
 
 	scanEntries(query: EntryScan): Promise<Entry[]> {
 		if (this.state !== "open") return Promise.reject(new Error("MemoryStorage is closed"));
-		const entries = [...this.entries.values()]
-			.filter((entry) => query.type === undefined || entry.type === query.type)
-			.filter((entry) => query.customType === undefined || entry.customType === query.customType)
-			.filter((entry) => query.fromSeq === undefined || entry.seq >= query.fromSeq)
-			.filter((entry) => query.toSeq === undefined || entry.seq <= query.toSeq)
-			.sort((left, right) => (query.order === "desc" ? right.seq - left.seq : left.seq - right.seq));
-		return Promise.resolve(query.limit === undefined ? entries : entries.slice(0, Math.max(0, query.limit)));
+		const limit = query.limit === undefined ? Number.POSITIVE_INFINITY : Math.max(0, Math.trunc(query.limit));
+		const entries: Entry[] = [];
+		const descending = query.order === "desc";
+		let index = descending ? this.entriesBySeq.length - 1 : 0;
+		while (index >= 0 && index < this.entriesBySeq.length && entries.length < limit) {
+			const entry = this.entriesBySeq[index]!;
+			if (
+				(query.type === undefined || entry.type === query.type) &&
+				(query.customType === undefined || entry.customType === query.customType) &&
+				(query.fromSeq === undefined || entry.seq >= query.fromSeq) &&
+				(query.toSeq === undefined || entry.seq <= query.toSeq)
+			) {
+				entries.push(entry);
+			}
+			index += descending ? -1 : 1;
+		}
+		return Promise.resolve(entries);
 	}
 
 	scanUsage(query: UsageScan): Promise<UsageRow[]> {
@@ -255,7 +269,7 @@ export class MemoryStorage implements Storage {
 	snapshot(): Promise<{ entries: Entry[]; registers: Register[] }> {
 		if (this.state !== "open") return Promise.reject(new Error("MemoryStorage is closed"));
 		const result = this.commitQueue.then(() => ({
-			entries: [...this.entries.values()],
+			entries: [...this.entriesBySeq],
 			registers: [...this.registers.values()],
 		}));
 		this.commitQueue = result.then(
@@ -286,6 +300,7 @@ export class MemoryStorage implements Storage {
 	): MemoryStorage {
 		const storage = new MemoryStorage(options);
 		storage.entries = snapshot.entries;
+		storage.entriesBySeq = [...snapshot.entries.values()].sort((left, right) => left.seq - right.seq);
 		storage.registers = snapshot.registers;
 		storage.usage = snapshot.usage;
 		storage.stats = snapshot.stats;

packages/agent/src/harness/session/testing/benchmark/datasets.ts

@@ -0,0 +1,33 @@
+export interface StorageBenchmarkDataset {
+	readonly name: string;
+	readonly entryCount: number;
+	readonly payloadBytes: number;
+	readonly lookupIds: readonly string[];
+	readonly leafId: string;
+}
+
+/** Package-internal deterministic id shared by dataset and transaction generation. */
+export function storageBenchmarkEntryId(index: number): string {
+	return `benchmark-entry-${index.toString().padStart(8, "0")}`;
+}
+
+function createDataset(scale: string, entryCount: number): StorageBenchmarkDataset {
+	const lookupCount = Math.min(100, entryCount);
+	return {
+		name: `synthetic linear branch: ${scale}, 256-byte payloads`,
+		entryCount,
+		payloadBytes: 256,
+		lookupIds: Array.from({ length: lookupCount }, (_, index) => {
+			const entryIndex = Math.floor((index * (entryCount - 1)) / Math.max(1, lookupCount - 1));
+			return storageBenchmarkEntryId(entryIndex);
+		}),
+		leafId: storageBenchmarkEntryId(entryCount - 1),
+	};
+}
+
+/** Deterministic synthetic linear branches shared by all storage measurements. */
+export const STORAGE_BENCHMARK_DATASETS: readonly StorageBenchmarkDataset[] = [
+	createDataset("1k entries", 1_000),
+	createDataset("10k entries", 10_000),
+	createDataset("100k entries", 100_000),
+];

packages/agent/src/harness/session/testing/benchmark/storage.ts

@@ -0,0 +1,147 @@
+import type { MessageEntry, NewEntry, Storage, Transaction } from "../../types.ts";
+import { STORAGE_BENCHMARK_DATASETS, type StorageBenchmarkDataset, storageBenchmarkEntryId } from "./datasets.ts";
+
+const MESSAGE_TIMESTAMP = 1_650_000_000_000;
+const SEED_BATCH_SIZE = 250;
+const WRITE_BASELINE_DATASET = STORAGE_BENCHMARK_DATASETS[0]!;
+
+function createEntry(index: number, payloadBytes: number): NewEntry<MessageEntry> {
+	const id = storageBenchmarkEntryId(index);
+	const prefix = `${id}:`;
+	return {
+		id,
+		parentId: index === 0 ? null : storageBenchmarkEntryId(index - 1),
+		type: "message",
+		message: {
+			role: "user",
+			content: [{ type: "text", text: prefix + "x".repeat(Math.max(0, payloadBytes - prefix.length)) }],
+			timestamp: MESSAGE_TIMESTAMP,
+		},
+	};
+}
+
+function createStorageBenchmarkTransaction(startIndex: number, entryCount: number, payloadBytes: number): Transaction {
+	return {
+		writes: Array.from({ length: entryCount }, (_, offset) => ({
+			kind: "entry" as const,
+			entry: createEntry(startIndex + offset, payloadBytes),
+		})),
+	};
+}
+
+/** Generates deterministic seed transactions without claiming a production data distribution. */
+export function* generateStorageBenchmarkSeedTransactions(dataset: StorageBenchmarkDataset): Generator<Transaction> {
+	for (let startIndex = 0; startIndex < dataset.entryCount; startIndex += SEED_BATCH_SIZE) {
+		yield createStorageBenchmarkTransaction(
+			startIndex,
+			Math.min(SEED_BATCH_SIZE, dataset.entryCount - startIndex),
+			dataset.payloadBytes,
+		);
+	}
+}
+
+/** Seeds one deterministic synthetic linear branch. */
+export async function seedStorageBenchmark(storage: Storage, dataset: StorageBenchmarkDataset): Promise<void> {
+	for (const transaction of generateStorageBenchmarkSeedTransactions(dataset)) await storage.commit(transaction);
+}
+
+/** A steady-state read operation run against a pre-seeded fixture. */
+interface StorageReadBenchmarkScenario {
+	readonly name: string;
+	expectedResult(dataset: StorageBenchmarkDataset): number;
+	run(storage: Storage, dataset: StorageBenchmarkDataset): Promise<number>;
+}
+
+/** A write operation run once against each independently prepared fixture. */
+interface StorageWriteBenchmarkScenario {
+	readonly name: string;
+	readonly writeCount: number;
+	prepare?(storage: Storage): Promise<void>;
+	run(storage: Storage): Promise<number>;
+}
+
+const singleEntryTransaction = createStorageBenchmarkTransaction(0, 1, 256);
+const hundredEntryTransaction = createStorageBenchmarkTransaction(0, 100, 256);
+const appendedEntryId = storageBenchmarkEntryId(WRITE_BASELINE_DATASET.entryCount);
+const mixedAppendTransaction: Transaction = {
+	writes: [
+		...createStorageBenchmarkTransaction(WRITE_BASELINE_DATASET.entryCount, 1, WRITE_BASELINE_DATASET.payloadBytes)
+			.writes,
+		{ kind: "register", op: "set", namespace: "lane.leaf", key: "main", value: appendedEntryId },
+		{
+			kind: "usage",
+			row: {
+				id: "benchmark-usage",
+				entryId: appendedEntryId,
+				adjustment: false,
+				usage: {
+					input: 1_000,
+					output: 250,
+					cacheRead: 500,
+					cacheWrite: 0,
+					totalTokens: 1_250,
+					cost: { input: 0.001, output: 0.001, cacheRead: 0.0001, cacheWrite: 0, total: 0.0021 },
+				},
+			},
+		},
+	],
+};
+
+/** Shared read scenarios. Returning a number ensures each result is consumed. */
+export const STORAGE_READ_BENCHMARK_SCENARIOS: readonly StorageReadBenchmarkScenario[] = [
+	{
+		name: "get 100 distributed entries",
+		expectedResult(dataset) {
+			return dataset.lookupIds.length;
+		},
+		async run(storage, dataset) {
+			return (await storage.getEntries([...dataset.lookupIds])).size;
+		},
+	},
+	{
+		name: "scan latest 50 entries",
+		expectedResult(dataset) {
+			return Math.min(50, dataset.entryCount);
+		},
+		async run(storage) {
+			return (await storage.scanEntries({ order: "desc", limit: 50 })).length;
+		},
+	},
+	{
+		name: "scan full branch structure",
+		expectedResult(dataset) {
+			return dataset.entryCount;
+		},
+		async run(storage, dataset) {
+			return (await storage.scanBranchStructure({ start: dataset.leafId, order: "newestFirst" })).length;
+		},
+	},
+];
+
+/** Shared writes. Every invocation receives equivalent pre-benchmark state. */
+export const STORAGE_WRITE_BENCHMARK_SCENARIOS: readonly StorageWriteBenchmarkScenario[] = [
+	{
+		name: "commit one message entry",
+		writeCount: 1,
+		async run(storage) {
+			return (await storage.commit(singleEntryTransaction)).seqs.length;
+		},
+	},
+	{
+		name: "commit 100 message entries",
+		writeCount: 100,
+		async run(storage) {
+			return (await storage.commit(hundredEntryTransaction)).seqs.length;
+		},
+	},
+	{
+		name: `commit mixed append (${WRITE_BASELINE_DATASET.name})`,
+		writeCount: 3,
+		prepare(storage) {
+			return seedStorageBenchmark(storage, WRITE_BASELINE_DATASET);
+		},
+		async run(storage) {
+			return (await storage.commit(mixedAppendTransaction)).seqs.length;
+		},
+	},
+];

packages/agent/src/harness/session/testing/conformance/session-repo.ts

@@ -1,7 +1,7 @@
 import { deepStrictEqual, rejects, strictEqual } from "node:assert/strict";
 import type { AssistantMessage, StopReason } from "@earendil-works/pi-ai";
 import type { LaneConfiguration, LaneState, UsageRow } from "../../types.ts";
-import type { SessionRepoConformanceCase, SessionRepoFixture, SessionRepoFixtureFactory } from "../types.ts";
+import type { ConformanceCase, SessionRepoFixture } from "../types.ts";
 
 const ROOT_ID = "00000000-0000-7000-8000-000000000001";
 const CHILD_ID = "00000000-0000-7000-8000-000000000002";
@@ -58,11 +58,11 @@ function usageRow(): Omit<UsageRow, "seq"> {
 }
 
 function createCase(
-	factory: SessionRepoFixtureFactory,
+	factory: () => Promise<SessionRepoFixture>,
 	group: string,
 	name: string,
 	test: (fixture: SessionRepoFixture) => Promise<void>,
-): SessionRepoConformanceCase {
+): ConformanceCase {
 	return {
 		group,
 		name,
@@ -74,9 +74,7 @@ function createCase(
 }
 
 /** Creates fresh, runner-independent cases for the SessionRepo contract. */
-export function createSessionRepoConformance(
-	factory: SessionRepoFixtureFactory,
-): readonly SessionRepoConformanceCase[] {
+export function createSessionRepoConformance(factory: () => Promise<SessionRepoFixture>): readonly ConformanceCase[] {
 	return [
 		createCase(
 			factory,

packages/agent/src/harness/session/testing/conformance/storage.ts

@@ -9,18 +9,18 @@ import type {
 	Register,
 	Transaction,
 } from "../../types.ts";
-import type { StorageConformanceCase, StorageFixture, StorageFixtureFactory } from "../types.ts";
+import type { ConformanceCase, StorageFixture } from "../types.ts";
 
 const MESSAGE_TIMESTAMP = 1_650_000_000_000;
 
 type ConformanceTest = (fixture: StorageFixture) => Promise<void>;
 
 function createCase(
-	factory: StorageFixtureFactory,
+	factory: () => Promise<StorageFixture>,
 	group: string,
 	name: string,
 	test: ConformanceTest,
-): StorageConformanceCase {
+): ConformanceCase {
 	return {
 		group,
 		name,
@@ -110,7 +110,7 @@ function sortRegisters(registers: Register[]): Register[] {
 }
 
 /** Creates fresh, runner-independent cases for the durable Storage contract. */
-export function createStorageConformance(factory: StorageFixtureFactory): readonly StorageConformanceCase[] {
+export function createStorageConformance(factory: () => Promise<StorageFixture>): readonly ConformanceCase[] {
 	return [
 		createCase(factory, "transactions", "commits mixed writes atomically in write order", async ({ storage }) => {
 			const result = await storage.commit({

packages/agent/src/harness/session/testing/index.ts

@@ -1,13 +1,16 @@
+export { STORAGE_BENCHMARK_DATASETS } from "./benchmark/datasets.ts";
+export {
+	generateStorageBenchmarkSeedTransactions,
+	STORAGE_READ_BENCHMARK_SCENARIOS,
+	STORAGE_WRITE_BENCHMARK_SCENARIOS,
+	seedStorageBenchmark,
+} from "./benchmark/storage.ts";
 export { createSessionRepoConformance } from "./conformance/session-repo.ts";
 export { createStorageConformance } from "./conformance/storage.ts";
-export type { RecordedCommitAttempt } from "./instrumented-storage.ts";
 export { InstrumentedStorage } from "./instrumented-storage.ts";
 export { StorageDecorator } from "./storage-decorator.ts";
 export type {
-	SessionRepoConformanceCase,
+	ConformanceCase,
 	SessionRepoFixture,
-	SessionRepoFixtureFactory,
-	StorageConformanceCase,
 	StorageFixture,
-	StorageFixtureFactory,
 } from "./types.ts";

packages/agent/src/harness/session/testing/instrumented-storage.ts

@@ -1,31 +1,11 @@
-import type {
-	CommitResult,
-	Entry,
-	EntryScan,
-	EntryStructure,
-	Register,
-	RegisterNamespace,
-	SessionStats,
-	Storage,
-	StorageBranchScan,
-	Transaction,
-	UsageRow,
-	UsageScan,
-} from "../types.ts";
-
-/** A transaction captured at commit admission. */
-export type RecordedCommitAttempt = Transaction;
+import type { CommitResult, Transaction } from "../types.ts";
+import { StorageDecorator } from "./storage-decorator.ts";
 
 /** Test-only transparent Storage decorator that records commit admission. */
-export class InstrumentedStorage implements Storage {
-	private readonly delegate: Storage;
+export class InstrumentedStorage extends StorageDecorator {
 	private readonly commitAttempts: Transaction[] = [];
 
-	constructor(delegate: Storage) {
-		this.delegate = delegate;
-	}
-
-	getCommitAttempts(): readonly RecordedCommitAttempt[] {
+	getCommitAttempts(): readonly Transaction[] {
 		return this.commitAttempts.slice();
 	}
 
@@ -33,50 +13,8 @@ export class InstrumentedStorage implements Storage {
 		this.commitAttempts.length = 0;
 	}
 
-	commit(transaction: Transaction): Promise<CommitResult> {
+	override commit(transaction: Transaction): Promise<CommitResult> {
 		this.commitAttempts.push(transaction);
 		return this.delegate.commit(transaction);
 	}
-
-	getEntries(ids: string[]): Promise<Map<string, Entry>> {
-		return this.delegate.getEntries(ids);
-	}
-
-	getRegister<TNamespace extends RegisterNamespace>(
-		namespace: TNamespace,
-		key: string,
-	): Promise<Register<TNamespace> | undefined> {
-		return this.delegate.getRegister(namespace, key);
-	}
-
-	listRegisters<TNamespace extends RegisterNamespace>(
-		namespace: TNamespace,
-		keyPrefix?: string,
-	): Promise<Register<TNamespace>[]> {
-		return this.delegate.listRegisters(namespace, keyPrefix);
-	}
-
-	scanBranch(query: StorageBranchScan): Promise<Entry[]> {
-		return this.delegate.scanBranch(query);
-	}
-
-	scanBranchStructure(query: StorageBranchScan): Promise<EntryStructure[]> {
-		return this.delegate.scanBranchStructure(query);
-	}
-
-	scanEntries(query: EntryScan): Promise<Entry[]> {
-		return this.delegate.scanEntries(query);
-	}
-
-	scanUsage(query: UsageScan): Promise<UsageRow[]> {
-		return this.delegate.scanUsage(query);
-	}
-
-	getStats(): Promise<SessionStats> {
-		return this.delegate.getStats();
-	}
-
-	close(): Promise<void> {
-		return this.delegate.close();
-	}
 }

packages/agent/src/harness/session/testing/types.ts

@@ -1,15 +1,12 @@
 import type { SessionRepo, Storage } from "../types.ts";
 
-/** A fresh backend storage instance owned by one conformance case. */
+/** A fresh backend storage instance owned by one test or benchmark case. */
 export interface StorageFixture extends AsyncDisposable {
 	readonly storage: Storage;
 }
 
-/** Creates an isolated storage fixture for one conformance case. */
-export type StorageFixtureFactory = () => Promise<StorageFixture>;
-
 /** A runner-independent conformance case that can be registered with any test framework. */
-export interface StorageConformanceCase {
+export interface ConformanceCase {
 	readonly group: string;
 	readonly name: string;
 	run(): Promise<void>;
@@ -20,9 +17,3 @@ export interface SessionRepoFixture extends AsyncDisposable {
 	readonly repo: SessionRepo;
 	readonly storageVersion: number;
 }
-
-/** Creates an isolated repository fixture for one conformance case. */
-export type SessionRepoFixtureFactory = () => Promise<SessionRepoFixture>;
-
-/** A runner-independent conformance case for the SessionRepo contract. */
-export type SessionRepoConformanceCase = StorageConformanceCase;

packages/agent/test/harness/memory-conformance.test.ts

@@ -0,0 +1,48 @@
+import { describe, it } from "vitest";
+import { MemorySessionRepo } from "../../src/harness/session/index.ts";
+import { MemoryStorage } from "../../src/harness/session/memory.ts";
+import {
+	type ConformanceCase,
+	createSessionRepoConformance,
+	createStorageConformance,
+	type SessionRepoFixture,
+	type StorageFixture,
+} from "../../src/harness/session/testing/index.ts";
+
+const NOW = 1_700_000_000_000;
+const STORAGE_VERSION = 1;
+
+function registerConformance(name: string, cases: readonly ConformanceCase[]): void {
+	describe(name, () => {
+		for (const group of new Set(cases.map((testCase) => testCase.group))) {
+			describe(group, () => {
+				for (const testCase of cases.filter((candidate) => candidate.group === group)) {
+					it(testCase.name, () => testCase.run());
+				}
+			});
+		}
+	});
+}
+
+registerConformance(
+	"MemoryStorage conformance",
+	createStorageConformance(() => {
+		const storage = new MemoryStorage({ now: () => NOW });
+		return Promise.resolve<StorageFixture>({
+			storage,
+			[Symbol.asyncDispose]: () => storage.close(),
+		});
+	}),
+);
+
+registerConformance(
+	"MemorySessionRepo conformance",
+	createSessionRepoConformance(() => {
+		const repo = new MemorySessionRepo({ now: () => NOW });
+		return Promise.resolve<SessionRepoFixture>({
+			repo,
+			storageVersion: STORAGE_VERSION,
+			[Symbol.asyncDispose]: () => repo.close(),
+		});
+	}),
+);

packages/agent/test/harness/memory-session-repo.test.ts

@@ -1,29 +1,7 @@
 import { describe, expect, it } from "vitest";
 import { MemorySessionRepo } from "../../src/harness/session/index.ts";
-import { createSessionRepoConformance, type SessionRepoFixture } from "../../src/harness/session/testing/index.ts";
 
 const NOW = 1_700_000_000_000;
-const STORAGE_VERSION = 1;
-
-const conformance = createSessionRepoConformance(() => {
-	const repo = new MemorySessionRepo({ now: () => NOW });
-	return Promise.resolve<SessionRepoFixture>({
-		repo,
-		storageVersion: STORAGE_VERSION,
-		[Symbol.asyncDispose]: () => repo.close(),
-	});
-});
-
-describe("MemorySessionRepo conformance", () => {
-	for (const group of new Set(conformance.map((testCase) => testCase.group))) {
-		describe(group, () => {
-			for (const testCase of conformance.filter((candidate) => candidate.group === group)) {
-				it(testCase.name, () => testCase.run());
-			}
-		});
-	}
-});
-
 function uuidTimestamp(id: string): number {
 	return Number.parseInt(id.replaceAll("-", "").slice(0, 12), 16);
 }

packages/agent/test/harness/memory-storage.test.ts

@@ -1,27 +1,8 @@
 import { describe, expect, it } from "vitest";
 import { MemoryStorage } from "../../src/harness/session/memory.ts";
-import { createStorageConformance, type StorageFixture } from "../../src/harness/session/testing/index.ts";
 
 const NOW = 1_700_000_000_000;
 
-const conformance = createStorageConformance(() => {
-	const storage = new MemoryStorage({ now: () => NOW });
-	return Promise.resolve<StorageFixture>({
-		storage,
-		[Symbol.asyncDispose]: () => storage.close(),
-	});
-});
-
-describe("MemoryStorage conformance", () => {
-	for (const group of new Set(conformance.map((testCase) => testCase.group))) {
-		describe(group, () => {
-			for (const testCase of conformance.filter((candidate) => candidate.group === group)) {
-				it(testCase.name, () => testCase.run());
-			}
-		});
-	}
-});
-
 describe("MemoryStorage", () => {
 	it("uses the injected clock once per transaction", async () => {
 		let timestamp = NOW;

packages/agent/vitest.benchmark.config.ts

@@ -0,0 +1,23 @@
+import { fileURLToPath } from "node:url";
+import { defineConfig } from "vitest/config";
+
+const telemetrySrcIndex = fileURLToPath(new URL("../telemetry/src/index.ts", import.meta.url));
+const aiSrcIndex = fileURLToPath(new URL("../ai/src/index.ts", import.meta.url));
+const aiSrcCompat = fileURLToPath(new URL("../ai/src/compat.ts", import.meta.url));
+
+export default defineConfig({
+	test: {
+		environment: "node",
+		benchmark: {
+			include: ["benchmark/session/**/*.bench.ts"],
+			reporters: ["verbose"],
+		},
+	},
+	resolve: {
+		alias: [
+			{ find: /^@earendil-works\/pi-telemetry$/, replacement: telemetrySrcIndex },
+			{ find: /^@earendil-works\/pi-ai$/, replacement: aiSrcIndex },
+			{ find: /^@earendil-works\/pi-ai\/compat$/, replacement: aiSrcCompat },
+		],
+	},
+});