From 43c4f6cf725b84196363de181a3d43402854b583 Mon Sep 17 00:00:00 2001 From: Cristina Poncela Cubeiro <140309543+cristinaponcela@users.noreply.github.com> Date: Tue, 18 Aug 2026 18:25:34 +0200 Subject: [PATCH] test(sqlite): timing benchmark --- .../benchmark/session/session-repo-targets.ts | 71 +++++++++++++++++++ .../benchmark/session/session-repo.bench.ts | 62 ++++++++++++++++ .../benchmark/session/storage-targets.ts | 47 ++++++++++++ .../benchmark/session/storage.bench.ts | 35 +++++++++ .../session-backends/sqlite-node/package.json | 1 + .../sqlite-node/vitest.benchmark.config.ts | 27 +++++++ 6 files changed, 243 insertions(+) create mode 100644 packages/session-backends/sqlite-node/benchmark/session/session-repo-targets.ts create mode 100644 packages/session-backends/sqlite-node/benchmark/session/session-repo.bench.ts create mode 100644 packages/session-backends/sqlite-node/benchmark/session/storage-targets.ts create mode 100644 packages/session-backends/sqlite-node/benchmark/session/storage.bench.ts create mode 100644 packages/session-backends/sqlite-node/vitest.benchmark.config.ts diff --git a/packages/session-backends/sqlite-node/benchmark/session/session-repo-targets.ts b/packages/session-backends/sqlite-node/benchmark/session/session-repo-targets.ts new file mode 100644 index 000000000..23c0664de --- /dev/null +++ b/packages/session-backends/sqlite-node/benchmark/session/session-repo-targets.ts @@ -0,0 +1,71 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT } from "@earendil-works/pi-agent-core"; +import type { Session, SessionCreateOptions, SessionMetadata, SessionRepo } from "@earendil-works/pi-agent-core"; +import { createNodeSqliteFactory, SqliteSessionRepo } from "../../src/index.ts"; +import type { BenchmarkTarget } from "../../../../agent/benchmark/session/benchmark.ts"; + +export interface SessionRepoBenchmarkFixture extends AsyncDisposable { + readonly repo: SessionRepo; +} + +const NOW = 1_700_000_000_000; + +class TrackingSessionRepo implements SessionRepo { + private readonly sessions = new Set>(); + + constructor(private readonly inner: SessionRepo) {} + + async create(options: SessionCreateOptions | undefined, context = BACKGROUND_CONTEXT): Promise> { + return this.track(await this.inner.create(options, context)); + } + + async open(metadata: TMetadata, context = BACKGROUND_CONTEXT): Promise> { + return this.track(await this.inner.open(metadata, context)); + } + + list(options: undefined, context = BACKGROUND_CONTEXT): Promise { + return this.inner.list(options, context); + } + + delete(metadata: TMetadata, context = BACKGROUND_CONTEXT): Promise { + return this.inner.delete(metadata, context); + } + + async fork(source: TMetadata, options = {}, context = BACKGROUND_CONTEXT): Promise> { + return this.track(await this.inner.fork(source, options, context)); + } + + async closeTrackedSessions(): Promise { + await Promise.allSettled([...this.sessions].map((session) => session.close(BACKGROUND_CONTEXT))); + this.sessions.clear(); + } + + private track(session: Session): Session { + this.sessions.add(session); + return session; + } +} + +export const sessionRepoBenchmarkTargets = [ + { + name: "sqlite", + async createFixture() { + const directory = await mkdtemp(join(tmpdir(), "pi-sqlite-session-repo-benchmark-")); + const sqliteRepo = new SqliteSessionRepo({ + directory, + databaseFactory: createNodeSqliteFactory(), + now: () => NOW, + }); + const repo = new TrackingSessionRepo(sqliteRepo); + return { + repo, + async [Symbol.asyncDispose]() { + await repo.closeTrackedSessions(); + await rm(directory, { recursive: true, force: true }); + }, + }; + }, + }, +] satisfies readonly BenchmarkTarget[]; diff --git a/packages/session-backends/sqlite-node/benchmark/session/session-repo.bench.ts b/packages/session-backends/sqlite-node/benchmark/session/session-repo.bench.ts new file mode 100644 index 000000000..015d7d996 --- /dev/null +++ b/packages/session-backends/sqlite-node/benchmark/session/session-repo.bench.ts @@ -0,0 +1,62 @@ +import { + SESSION_REPO_CATALOG_BENCHMARK_DATASETS, + SESSION_REPO_CATALOG_READ_BENCHMARK_SCENARIOS, + SESSION_REPO_CATALOG_WRITE_BENCHMARK_SCENARIOS, + SESSION_REPO_FORK_BENCHMARK_DATASETS, + SESSION_REPO_FORK_WRITE_BENCHMARK_SCENARIOS, + seedSessionRepoCatalogBenchmark, + seedSessionRepoForkBenchmark, +} from "@earendil-works/pi-agent-core/session/testing"; +import { registerReadBenchmarks, registerWriteBenchmarks } from "../../../../agent/benchmark/session/benchmark.ts"; +import { sessionRepoBenchmarkTargets } from "./session-repo-targets.ts"; + +await registerReadBenchmarks({ + datasets: SESSION_REPO_CATALOG_BENCHMARK_DATASETS, + targets: sessionRepoBenchmarkTargets, + scenarios: SESSION_REPO_CATALOG_READ_BENCHMARK_SCENARIOS, + async prepare(fixture, dataset) { + await seedSessionRepoCatalogBenchmark(fixture.repo, dataset); + }, + getSubject(fixture) { + return fixture.repo; + }, +}); + +await registerWriteBenchmarks({ + targets: sessionRepoBenchmarkTargets, + scenarios: SESSION_REPO_CATALOG_WRITE_BENCHMARK_SCENARIOS, + prepare(fixture, scenario) { + return scenario.prepare(fixture.repo); + }, + expectedResult(scenario) { + return scenario.expectedResult; + }, + run(operation) { + return operation.run(); + }, +}); + +const forkBenchmarks = SESSION_REPO_FORK_BENCHMARK_DATASETS.flatMap((dataset) => + SESSION_REPO_FORK_WRITE_BENCHMARK_SCENARIOS.map((scenario) => ({ + name: `${scenario.name} (${dataset.name})`, + dataset, + scenario, + })), +); + +await registerWriteBenchmarks({ + targets: sessionRepoBenchmarkTargets, + scenarios: forkBenchmarks, + async prepare(fixture, benchmark) { + return { + repo: fixture.repo, + source: await seedSessionRepoForkBenchmark(fixture.repo, benchmark.dataset), + }; + }, + expectedResult(benchmark) { + return benchmark.scenario.expectedResult(benchmark.dataset); + }, + run(subject, benchmark) { + return benchmark.scenario.run(subject.repo, subject.source, benchmark.dataset); + }, +}); diff --git a/packages/session-backends/sqlite-node/benchmark/session/storage-targets.ts b/packages/session-backends/sqlite-node/benchmark/session/storage-targets.ts new file mode 100644 index 000000000..f044ef958 --- /dev/null +++ b/packages/session-backends/sqlite-node/benchmark/session/storage-targets.ts @@ -0,0 +1,47 @@ +import { BACKGROUND_CONTEXT } from "@earendil-works/pi-agent-core"; +import type { StorageFixture } from "@earendil-works/pi-agent-core/session/testing"; +import { createNodeSqliteFactory, SQLITE_STORAGE_VERSION, SqliteStorage, sql } from "../../src/index.ts"; +import { applyInitialSchema } from "../../src/sqlite/migrations.ts"; +import type { BenchmarkTarget } from "../../../../agent/benchmark/session/benchmark.ts"; + +const NOW = 1_700_000_000_000; +const EMPTY_USAGE = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, +}; + +export const storageBenchmarkTargets = [ + { + name: "sqlite", + async createFixture() { + const db = await createNodeSqliteFactory().open(":memory:"); + try { + db.exec("PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000;"); + await applyInitialSchema(db); + sql`INSERT INTO session + (created_at, parent_session_id, storage_version, metadata, message_count, usage_payload, next_seq) + VALUES (${NOW}, ${null}, ${SQLITE_STORAGE_VERSION}, ${null}, ${0}, ${JSON.stringify(EMPTY_USAGE)}, ${1})`.run( + db, + ); + const storage = new SqliteStorage(db, { now: () => NOW }); + return { + storage, + async [Symbol.asyncDispose]() { + try { + await storage.close(BACKGROUND_CONTEXT); + } finally { + db.close(); + } + }, + } satisfies StorageFixture; + } catch (error) { + db.close(); + throw error; + } + }, + }, +] satisfies readonly BenchmarkTarget[]; diff --git a/packages/session-backends/sqlite-node/benchmark/session/storage.bench.ts b/packages/session-backends/sqlite-node/benchmark/session/storage.bench.ts new file mode 100644 index 000000000..a5d8bac61 --- /dev/null +++ b/packages/session-backends/sqlite-node/benchmark/session/storage.bench.ts @@ -0,0 +1,35 @@ +import { + STORAGE_BENCHMARK_DATASETS, + STORAGE_READ_BENCHMARK_SCENARIOS, + STORAGE_WRITE_BENCHMARK_SCENARIOS, + seedStorageBenchmark, +} from "@earendil-works/pi-agent-core/session/testing"; +import { registerReadBenchmarks, registerWriteBenchmarks } from "../../../../agent/benchmark/session/benchmark.ts"; +import { storageBenchmarkTargets } from "./storage-targets.ts"; + +await registerReadBenchmarks({ + datasets: STORAGE_BENCHMARK_DATASETS, + targets: storageBenchmarkTargets, + scenarios: STORAGE_READ_BENCHMARK_SCENARIOS, + prepare(fixture, dataset) { + return seedStorageBenchmark(fixture.storage, dataset); + }, + getSubject(fixture) { + return fixture.storage; + }, +}); + +await registerWriteBenchmarks({ + targets: storageBenchmarkTargets, + scenarios: STORAGE_WRITE_BENCHMARK_SCENARIOS, + async prepare(fixture, scenario) { + await scenario.prepare?.(fixture.storage); + return fixture.storage; + }, + expectedResult(scenario) { + return scenario.writeCount; + }, + run(storage, scenario) { + return scenario.run(storage); + }, +}); diff --git a/packages/session-backends/sqlite-node/package.json b/packages/session-backends/sqlite-node/package.json index 5d4af8ad9..8023c67b1 100644 --- a/packages/session-backends/sqlite-node/package.json +++ b/packages/session-backends/sqlite-node/package.json @@ -19,6 +19,7 @@ "scripts": { "clean": "node -e \"import('node:fs/promises').then(fs=>fs.rm('dist',{recursive:true,force:true}))\"", "build": "tsgo -p tsconfig.build.json && node scripts/copy-migrations.mjs", + "bench:session:timing": "vitest bench --config vitest.benchmark.config.ts", "test": "vitest --run", "coverage": "vitest --run --coverage", "prepublishOnly": "npm run clean && npm run build" diff --git a/packages/session-backends/sqlite-node/vitest.benchmark.config.ts b/packages/session-backends/sqlite-node/vitest.benchmark.config.ts new file mode 100644 index 000000000..5b4c29a8b --- /dev/null +++ b/packages/session-backends/sqlite-node/vitest.benchmark.config.ts @@ -0,0 +1,27 @@ +import { fileURLToPath } from "node:url"; +import { defineConfig } from "vitest/config"; + +const telemetryIndex = fileURLToPath(new URL("../../telemetry/src/index.ts", import.meta.url)); +const aiIndex = fileURLToPath(new URL("../../ai/src/index.ts", import.meta.url)); +const agentIndex = fileURLToPath(new URL("../../agent/src/index.ts", import.meta.url)); +const agentSessionTesting = fileURLToPath( + new URL("../../agent/src/harness/session/testing/index.ts", import.meta.url), +); + +export default defineConfig({ + test: { + environment: "node", + benchmark: { + include: ["benchmark/session/**/*.bench.ts"], + reporters: ["verbose"], + }, + }, + resolve: { + alias: [ + { find: /^@earendil-works\/pi-telemetry$/, replacement: telemetryIndex }, + { find: /^@earendil-works\/pi-agent-core\/session\/testing$/, replacement: agentSessionTesting }, + { find: /^@earendil-works\/pi-agent-core$/, replacement: agentIndex }, + { find: /^@earendil-works\/pi-ai$/, replacement: aiIndex }, + ], + }, +});