// A read-only report of what a p2 data directory holds, to check that nothing was lost across a deploy that changes // no data (the move to Chord v2): sessions with their conversations and newest entries, subagents, shares, artifacts, // uploads and downloads, and the names of environments and port forwards. Never secrets: no tokens, cookies, keys, // forward addresses or message text. Every SQLite file is opened read-only, so it runs beside a live server. // // node --import ./src/source.ts scripts/data-report.ts [--data ] > before.json // node --import ./src/source.ts scripts/data-report.ts [--data ] --compare before.json // // `--compare` prints each difference as `path: before -> after` and exits 1 if there is one. import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; /** What the report says about one session. */ export type SessionReport = { readonly title: string; /** `env:cwd` on an SSH environment, else the cwd. */ readonly location: string; readonly archived: boolean; /** The catalog's last activity (ms since the epoch). */ readonly updatedAt: number; /** Per conversation ID: its number of entries and its newest entry ID. */ readonly conversations: Readonly>; readonly subagents: readonly { readonly name: string; readonly location?: string; readonly archived: boolean }[]; /** The modes of its share links, and of the guests' grants (removed ones left out). */ readonly shares: { readonly links: readonly string[]; readonly guests: readonly string[] }; /** Per artifact name, its versions. */ readonly artifacts: Readonly>; readonly uploads: number; readonly downloads: number; }; export type DataReport = { readonly sessions: Readonly>; readonly envs: readonly string[]; readonly forwards: readonly { readonly port: number; readonly name?: string; readonly public: boolean }[]; }; /** Reports the data directory `dataDir`. */ export function dataReport(dataDir: string): DataReport { const catalog = rows( join(dataDir, "catalog.sqlite"), "SELECT id,title,cwd,env,updatedAt,archivedAt FROM sessions ORDER BY id", ); const children = new Map( rows(join(dataDir, "subagents.sqlite"), "SELECT session,value FROM children").map((row) => [ String(row.session), JSON.parse(String(row.value)) as StoredChild[], ]), ); const shares = sharesBySession(join(dataDir, "shares.sqlite")); const sessions: Record = {}; for (const row of catalog) { const id = String(row.id); sessions[id] = { title: String(row.title), location: located(row.env === null ? null : String(row.env), String(row.cwd)), archived: row.archivedAt !== null, updatedAt: Number(row.updatedAt), conversations: conversationsOf(join(dataDir, "sessions", `${id}.sqlite`)), subagents: (children.get(id) ?? []) .filter((child) => child.kind !== "agent") .map((child) => ({ name: child.name, ...(child.location ? { location: located(child.location.env, child.location.cwd) } : child.cwd === undefined ? {} : { location: child.cwd }), archived: child.archived === true, })) .sort((a, b) => a.name.localeCompare(b.name)), shares: shares.get(id) ?? { links: [], guests: [] }, artifacts: artifactsOf(join(dataDir, "artifacts", id)), uploads: entriesIn(join(dataDir, "uploads", id)).length, downloads: entriesIn(join(dataDir, "downloads", id)).filter((download) => existsSync(join(dataDir, "downloads", id, download, "metadata.json")), ).length, }; } return { sessions, envs: Object.keys(jsonFile(join(dataDir, "envs.json")) ?? {}).sort(), forwards: Object.values((jsonFile(join(dataDir, "forwards.json")) ?? {}) as Record) .map((forward) => ({ port: forward.port, ...(forward.name === undefined ? {} : { name: forward.name }), public: forward.public === true, })) .sort((a, b) => a.port - b.port), }; } /** A `ChildStore` row as `subagents.sqlite` keeps it (version 1 rows have an own `cwd`, later ones a `location`). */ type StoredChild = { readonly name: string; readonly kind?: "agent"; readonly archived?: boolean; readonly cwd?: string; readonly location?: { readonly env: string | null; readonly cwd: string }; }; type StoredForward = { readonly port: number; readonly name?: string; readonly public?: boolean }; type Row = Record; /** The rows of `sql` in the SQLite file `path`, opened read-only; none when the file does not exist. */ function rows(path: string, sql: string): Row[] { if (!existsSync(path)) return []; const db = new DatabaseSync(path, { readOnly: true }); try { return db.prepare(sql).all() as Row[]; } finally { db.close(); } } const located = (env: string | null, cwd: string): string => (env ? `${env}:${cwd}` : cwd); /** * The links' and the current guests' modes of every shared session. Version 1 of `shares.sqlite` (one session per * guest, through its link) reports as version 2's grants do, so a report from before its migration compares equal. */ function sharesBySession(path: string): Map { const shares = new Map(); const of = (session: string) => { const entry = shares.get(session) ?? { links: [], guests: [] }; shares.set(session, entry); return entry; }; for (const row of rows(path, "SELECT session,mode FROM links ORDER BY session,mode")) of(String(row.session)).links.push(String(row.mode)); const grants = rows(path, "SELECT 1 FROM sqlite_master WHERE type='table' AND name='grants'").length ? "SELECT session,mode FROM grants WHERE removed=0 ORDER BY session,mode" : "SELECT session,mode FROM guests JOIN links ON links.id=guests.linkId WHERE removed=0 ORDER BY session,mode"; for (const row of rows(path, grants)) of(String(row.session)).guests.push(String(row.mode)); return shares; } /** A session's conversations from its Durable store: entries and the newest entry ID of each. */ function conversationsOf(path: string): Record { const conversations: Record = {}; for (const row of rows( path, "SELECT conversation_id AS id, count(*) AS entries, max(id) AS newest FROM entries GROUP BY conversation_id ORDER BY conversation_id", )) conversations[String(row.id)] = { entries: Number(row.entries), newest: Number(row.newest) }; return conversations; } /** The artifacts of a session's directory (`//version.json`): their number of versions. */ function artifactsOf(directory: string): Record { const artifacts: Record = {}; for (const name of entriesIn(directory)) artifacts[name] = entriesIn(join(directory, name)).filter((version) => existsSync(join(directory, name, version, "version.json")), ).length; return artifacts; } /** The names in `directory` (sorted, temporary files left out), none when it does not exist. */ function entriesIn(directory: string): string[] { if (!existsSync(directory) || !statSync(directory).isDirectory()) return []; return readdirSync(directory) .filter((name) => !name.startsWith(".")) .sort(); } function jsonFile(path: string): Record | undefined { return existsSync(path) ? (JSON.parse(readFileSync(path, "utf8")) as Record) : undefined; } /** The differences between two reports, as `path: before -> after` lines. */ export function compareReports(before: unknown, after: unknown, path = ""): string[] { if (JSON.stringify(before) === JSON.stringify(after)) return []; const objects = before !== null && after !== null && typeof before === "object" && typeof after === "object" && !Array.isArray(before) && !Array.isArray(after); if (!objects) return [`${path || "/"}: ${JSON.stringify(before)} -> ${JSON.stringify(after)}`]; const keys = [...new Set([...Object.keys(before), ...Object.keys(after)])].sort(); return keys.flatMap((key) => compareReports( (before as Record)[key], (after as Record)[key], `${path}/${key}`, ), ); } function main(args: readonly string[]): void { const option = (name: string) => { const index = args.indexOf(name); return index === -1 ? undefined : args[index + 1]; }; const dataDir = option("--data") ?? process.env.P2_DATA_DIR ?? join(homedir(), ".pi", "p2"); const report = dataReport(dataDir); const compare = option("--compare"); if (compare === undefined) { console.log(JSON.stringify(report, null, "\t")); return; } const differences = compareReports(JSON.parse(readFileSync(compare, "utf8")), report); for (const line of differences) console.log(line); console.log(differences.length ? `${differences.length} differences` : "no differences"); if (differences.length) process.exitCode = 1; } if (import.meta.main) main(process.argv.slice(2));