mirror of
https://github.com/badlogic/pi-mono.git
synced 2026-11-05 23:37:25 +00:00
feat(durable): replicate storages as a commit log
Every built-in storage implements ReplayableStorage: head(), replay() at an explicit sequence, and export() of the current state as a log that replays to the same reads. SQLite schema version 3 adds sequence indexes so export streams. @earendil-works/pi-durable/sync adds ReplicatingStorage, which appends each committed batch to an outbox with fork copies resolved, compactRecords(), and versioned JSON log lines.
This commit is contained in:
Armin Ronacher
parent
89e309581b
commit
4c25119f54
24 changed files with 1586 additions and 33 deletions
214
packages/durable/test/replay-support.ts
Normal file
214
packages/durable/test/replay-support.ts
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
import type { JsonValue } from "@earendil-works/chord";
|
||||
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
|
||||
import { idFromNumber, seqFromNumber } from "../src/ids.ts";
|
||||
import {
|
||||
type ConversationId,
|
||||
type DocumentCreate,
|
||||
type DocumentId,
|
||||
type DocumentPoint,
|
||||
type DocumentRecord,
|
||||
type EntryId,
|
||||
type LogRecord,
|
||||
type LogWrite,
|
||||
type ReplayableStorage,
|
||||
ROOT_CONVERSATION_ID,
|
||||
type Storage,
|
||||
type StorageWrite,
|
||||
type SubmissionId,
|
||||
type SubmissionRecord,
|
||||
type TaskId,
|
||||
type TaskRecord,
|
||||
} from "../src/types.ts";
|
||||
|
||||
const context = BACKGROUND_CONTEXT;
|
||||
type StoredTask = TaskRecord<JsonValue, JsonValue, JsonValue>;
|
||||
|
||||
export const conversationId = (id: number) => idFromNumber<ConversationId>(id);
|
||||
export const entryId = (id: number) => idFromNumber<EntryId>(id);
|
||||
export const taskId = (id: number) => idFromNumber<TaskId<JsonValue>>(id);
|
||||
export const documentId = (id: number) => idFromNumber<DocumentId>(id);
|
||||
|
||||
export function task(id: number, status: "pending" | "running" | "terminal", phase = "work"): StoredTask {
|
||||
const base = {
|
||||
id: taskId(id),
|
||||
conversationId: ROOT_CONVERSATION_ID,
|
||||
kind: "test.task",
|
||||
version: 1,
|
||||
input: { id },
|
||||
background: false,
|
||||
abortRequested: false,
|
||||
};
|
||||
return status === "terminal"
|
||||
? { ...base, state: { status, outcome: { status: "completed", result: { phase } } } }
|
||||
: { ...base, state: { status, checkpoint: { phase } } };
|
||||
}
|
||||
|
||||
function submission(id: number, status: "queued" | "placed", entry?: number): SubmissionRecord {
|
||||
return status === "queued"
|
||||
? { id: idFromNumber<SubmissionId>(id), conversationId: ROOT_CONVERSATION_ID, type: "input", status }
|
||||
: {
|
||||
id: idFromNumber<SubmissionId>(id),
|
||||
conversationId: ROOT_CONVERSATION_ID,
|
||||
type: "input",
|
||||
status,
|
||||
entry: entryId(entry!),
|
||||
};
|
||||
}
|
||||
|
||||
export const entry = (id: number, conversation = ROOT_CONVERSATION_ID, extra: object = {}): LogWrite => ({
|
||||
type: "entry",
|
||||
value: { id: entryId(id), conversationId: conversation, kind: "message", data: { id }, ...extra },
|
||||
});
|
||||
|
||||
export const sessionDoc = (id: number, kind = "test.session"): DocumentCreate => ({
|
||||
id: documentId(id),
|
||||
kind,
|
||||
scope: { kind: "session" },
|
||||
});
|
||||
export const rewindableDoc = (
|
||||
id: number,
|
||||
conversation = ROOT_CONVERSATION_ID,
|
||||
kind = "test.rewindable",
|
||||
): DocumentCreate => ({
|
||||
id: documentId(id),
|
||||
kind,
|
||||
scope: { kind: "conversation", conversationId: conversation },
|
||||
history: "rewindable",
|
||||
fork: "asOf",
|
||||
});
|
||||
export const latestDoc = (id: number, conversation = ROOT_CONVERSATION_ID, kind = "test.latest"): DocumentCreate => ({
|
||||
id: documentId(id),
|
||||
kind,
|
||||
scope: { kind: "conversation", conversationId: conversation },
|
||||
history: "latest",
|
||||
fork: "current",
|
||||
});
|
||||
|
||||
export const create = (record: DocumentCreate, value: Record<string, JsonValue>, version = 1): LogWrite => ({
|
||||
type: "document.create",
|
||||
record,
|
||||
content: { kind: "base", version, value },
|
||||
});
|
||||
export const delta = (id: number, path: string, value: JsonValue, version = 1): LogWrite => ({
|
||||
type: "document.change",
|
||||
id: documentId(id),
|
||||
content: { kind: "delta", version, ops: [["s", [path], value]] },
|
||||
});
|
||||
export const base = (id: number, value: Record<string, JsonValue>, version = 1): LogWrite => ({
|
||||
type: "document.change",
|
||||
id: documentId(id),
|
||||
content: { kind: "base", version, value },
|
||||
});
|
||||
|
||||
/**
|
||||
* A history that exercises everything export has to preserve: reclaimed current-only content, rewindable history
|
||||
* across a version transition, task documents, fork copies, retire and recreate, an empty lifetime, upserted records,
|
||||
* and a head that no stamp records.
|
||||
*/
|
||||
export async function seed(storage: ReplayableStorage): Promise<void> {
|
||||
const batches: StorageWrite[][] = [
|
||||
[{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }],
|
||||
[
|
||||
entry(2),
|
||||
entry(3),
|
||||
{ type: "task", value: task(4, "pending") },
|
||||
{ type: "submission", value: submission(5, "queued") },
|
||||
create(sessionDoc(6), { n: 0 }),
|
||||
create(rewindableDoc(7), { a: 1 }),
|
||||
],
|
||||
[delta(6, "n", 1), delta(7, "b", 1), { type: "task", value: task(4, "running") }, entry(8)],
|
||||
[base(6, { n: 5 }), base(7, { a: 2 }, 2), { type: "submission", value: submission(5, "placed", 8) }],
|
||||
[
|
||||
create({ id: documentId(9), kind: "test.task-doc", scope: { kind: "task", taskId: taskId(4) } }, { t: 1 }),
|
||||
create(latestDoc(10), { x: 1 }),
|
||||
],
|
||||
[
|
||||
{
|
||||
type: "conversation",
|
||||
value: { id: conversationId(11), parent: { conversationId: ROOT_CONVERSATION_ID, at: entryId(8) } },
|
||||
},
|
||||
{
|
||||
type: "document.copy",
|
||||
record: rewindableDoc(12, conversationId(11)),
|
||||
source: { id: documentId(7), at: seqFromNumber(3) },
|
||||
},
|
||||
{
|
||||
type: "document.copy",
|
||||
record: latestDoc(13, conversationId(11)),
|
||||
source: { id: documentId(10), at: "current" },
|
||||
},
|
||||
entry(14, conversationId(11), { head: entryId(14) }),
|
||||
],
|
||||
[base(10, { x: 2 }), delta(7, "c", 1, 2)],
|
||||
[
|
||||
{ type: "task", value: task(4, "terminal") },
|
||||
delta(9, "t", 2),
|
||||
{ type: "document.retire", id: documentId(9) },
|
||||
delta(10, "y", 1),
|
||||
],
|
||||
[
|
||||
{ type: "document.retire", id: documentId(7) },
|
||||
create(rewindableDoc(15), { a: 3 }),
|
||||
create(latestDoc(16, ROOT_CONVERSATION_ID, "test.flash"), { gone: true }),
|
||||
{ type: "document.retire", id: documentId(16) },
|
||||
delta(13, "copied", true),
|
||||
],
|
||||
[{ type: "task", value: task(17, "pending") }],
|
||||
];
|
||||
for (const batch of batches) await storage.commit(batch, context);
|
||||
}
|
||||
|
||||
type Reads = Record<string, unknown>;
|
||||
|
||||
/** Every read a Session can make, at every point of the history, as one comparable value. */
|
||||
export async function reads(storage: Storage & Pick<ReplayableStorage, "head">): Promise<Reads> {
|
||||
const head = await storage.head(context);
|
||||
const out: Reads = { head };
|
||||
const conversations = (await storage.scanConversations({}, 1000, undefined, context)).items;
|
||||
out.conversations = conversations;
|
||||
for (const conversation of conversations) {
|
||||
const entries = (
|
||||
await storage.scanEntries({ conversationId: conversation.id, order: "ascending" }, 1000, undefined, context)
|
||||
).items;
|
||||
out[`entries:${conversation.id}`] = await Promise.all(entries.map((item) => storage.entry(item.id, context)));
|
||||
out[`head-marker:${conversation.id}`] = await storage.findLatestHeadMarker(conversation.id, undefined, context);
|
||||
}
|
||||
const tasks = (await storage.scanTasks({}, 1000, undefined, context)).items;
|
||||
out.tasks = tasks;
|
||||
out.submissions = (await storage.scanSubmissions({}, 1000, undefined, context)).items;
|
||||
out.submissionByStatus = (await storage.scanSubmissions({ status: "placed" }, 1000, undefined, context)).items;
|
||||
out.tasksByStatus = (await storage.scanTasks({ status: "terminal" }, 1000, undefined, context)).items;
|
||||
|
||||
const scopes: DocumentRecord["scope"][] = [
|
||||
{ kind: "session" },
|
||||
...conversations.map((conversation) => ({ kind: "conversation" as const, conversationId: conversation.id })),
|
||||
...tasks.map((item) => ({ kind: "task" as const, taskId: item.id })),
|
||||
];
|
||||
const points: DocumentPoint[] = ["current"];
|
||||
for (let seq = 1; seq <= (head ?? 0); seq++) points.push(seqFromNumber(seq));
|
||||
for (const scope of scopes) {
|
||||
for (const at of points) {
|
||||
const documents = (await storage.scanDocuments({ scope, at }, 1000, undefined, context)).items;
|
||||
out[`documents:${JSON.stringify(scope)}:${at}`] = documents;
|
||||
for (const record of documents) {
|
||||
out[`document:${record.id}:${at}`] = await storage
|
||||
.document(record.id, at, context)
|
||||
.catch((error: unknown) => ({ rejected: error instanceof Error ? error.name : "unknown" }));
|
||||
const address = {
|
||||
kind: record.kind,
|
||||
scope: record.scope,
|
||||
...(record.key === undefined ? {} : { key: record.key }),
|
||||
};
|
||||
out[`find:${record.id}:${at}`] = await storage.findDocument(address, at, context);
|
||||
}
|
||||
}
|
||||
}
|
||||
out.nextId = await storage.mintId();
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function collect(log: AsyncIterable<LogRecord>): Promise<LogRecord[]> {
|
||||
const records: LogRecord[] = [];
|
||||
for await (const record of log) records.push(record);
|
||||
return records;
|
||||
}
|
||||
|
|
@ -15,7 +15,7 @@ describe("Durable Object SQLite adapter", () => {
|
|||
it("keeps schema names unprefixed by default", async () => {
|
||||
const sqlite = new DatabaseSync(":memory:");
|
||||
await openDurableObjectSqliteStorage(durableObjectStorage(sqlite));
|
||||
expect(sqlite.prepare("SELECT version FROM durable_schema").get()).toEqual({ version: 2 });
|
||||
expect(sqlite.prepare("SELECT version FROM durable_schema").get()).toEqual({ version: 3 });
|
||||
expect(sqlite.prepare("SELECT name FROM sqlite_master WHERE name = 'pi_durable_schema'").get()).toBeUndefined();
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ describe("durable SQLite migrations", () => {
|
|||
await storage.close(BACKGROUND_CONTEXT);
|
||||
const migrated = await openNodeSqliteDatabase(path);
|
||||
try {
|
||||
expect(await migrated.get("SELECT version FROM durable_schema WHERE singleton = 1")).toEqual({ version: 2 });
|
||||
expect(await migrated.get("SELECT version FROM durable_schema WHERE singleton = 1")).toEqual({ version: 3 });
|
||||
expect(await migrated.get("SELECT status, record FROM tasks WHERE id = 5")).toEqual({
|
||||
status: "waiting",
|
||||
record: JSON.stringify(waiting),
|
||||
|
|
|
|||
198
packages/durable/test/storage-replay.test.ts
Normal file
198
packages/durable/test/storage-replay.test.ts
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
/// <reference lib="dom" />
|
||||
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
|
||||
import { IDBKeyRange as FakeIDBKeyRange, IDBFactory } from "fake-indexeddb";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { seqFromNumber } from "../src/ids.ts";
|
||||
import { IndexedDbStorage } from "../src/storage/indexeddb.ts";
|
||||
import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts";
|
||||
import { MemoryStorage } from "../src/storage/memory.ts";
|
||||
import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts";
|
||||
import { type LogRecord, type ReplayableStorage, ROOT_CONVERSATION_ID, type StorageWrite } from "../src/types.ts";
|
||||
import { collect, create, delta, documentId, entry, entryId, latestDoc, reads, seed, task } from "./replay-support.ts";
|
||||
|
||||
Object.defineProperty(globalThis, "IDBKeyRange", { configurable: true, value: FakeIDBKeyRange });
|
||||
|
||||
const context = BACKGROUND_CONTEXT;
|
||||
|
||||
const cleanups: (() => Promise<void>)[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
for (const cleanup of cleanups.splice(0).reverse()) await cleanup();
|
||||
});
|
||||
|
||||
async function temporaryDirectory(): Promise<string> {
|
||||
const directory = await mkdtemp(join(tmpdir(), "pi-durable-replay-"));
|
||||
cleanups.push(() => rm(directory, { recursive: true, force: true }));
|
||||
return directory;
|
||||
}
|
||||
|
||||
type Backend = {
|
||||
readonly name: string;
|
||||
open(): Promise<{ readonly storage: ReplayableStorage; reopen(): Promise<ReplayableStorage> }>;
|
||||
};
|
||||
|
||||
function track(storage: ReplayableStorage): ReplayableStorage {
|
||||
cleanups.push(() => storage.close(context).catch(() => {}));
|
||||
return storage;
|
||||
}
|
||||
|
||||
const backends: readonly Backend[] = [
|
||||
{
|
||||
name: "memory",
|
||||
async open() {
|
||||
const storage = new MemoryStorage();
|
||||
return { storage, reopen: async () => storage };
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "sqlite",
|
||||
async open() {
|
||||
const path = join(await temporaryDirectory(), "session.sqlite");
|
||||
let storage = track(await openNodeSqliteStorage(path));
|
||||
return {
|
||||
storage,
|
||||
reopen: async () => {
|
||||
await storage.close(context);
|
||||
storage = track(await openNodeSqliteStorage(path));
|
||||
return storage;
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "jsonl",
|
||||
async open() {
|
||||
const directory = await temporaryDirectory();
|
||||
let storage = track(await openNodeJsonlStorage(directory, context));
|
||||
return {
|
||||
storage,
|
||||
reopen: async () => {
|
||||
await storage.close(context);
|
||||
storage = track(await openNodeJsonlStorage(directory, context));
|
||||
return storage;
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "indexeddb",
|
||||
async open() {
|
||||
const indexedDB = new IDBFactory();
|
||||
let storage = track(await IndexedDbStorage.open("session", { indexedDB }));
|
||||
return {
|
||||
storage,
|
||||
reopen: async () => {
|
||||
await storage.close(context);
|
||||
storage = track(await IndexedDbStorage.open("session", { indexedDB }));
|
||||
return storage;
|
||||
},
|
||||
};
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
describe("export and replay", () => {
|
||||
for (const source of backends) {
|
||||
for (const target of backends) {
|
||||
it(`replays ${source.name} into ${target.name} with the same reads`, async () => {
|
||||
const opened = await source.open();
|
||||
await seed(opened.storage);
|
||||
const reopened = await opened.reopen();
|
||||
const log = await collect(reopened.export(context));
|
||||
expect(log.at(-1)?.seq).toBe(await reopened.head(context));
|
||||
for (const record of log) {
|
||||
expect(record.writes.some((write) => (write as StorageWrite).type === "document.copy")).toBe(false);
|
||||
}
|
||||
|
||||
const replica = (await target.open()).storage;
|
||||
for (const record of log) await replica.replay(record, context);
|
||||
expect(await reads(replica)).toEqual(await reads(reopened));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const backend of backends) {
|
||||
describe(backend.name, () => {
|
||||
it("exports nothing from an empty storage", async () => {
|
||||
const { storage } = await backend.open();
|
||||
expect(await storage.head(context)).toBeUndefined();
|
||||
expect(await collect(storage.export(context))).toEqual([]);
|
||||
});
|
||||
|
||||
it("puts records without a sequence into the first record and ends at the head", async () => {
|
||||
const { storage } = await backend.open();
|
||||
await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context);
|
||||
await storage.commit([{ type: "task", value: task(2, "pending") }], context);
|
||||
expect(await collect(storage.export(context))).toEqual([
|
||||
{
|
||||
seq: 2,
|
||||
writes: [
|
||||
{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } },
|
||||
{ type: "task", value: task(2, "pending") },
|
||||
],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("replays at explicit sequences with gaps and continues after them", async () => {
|
||||
const { storage } = await backend.open();
|
||||
await storage.replay(
|
||||
{ seq: seqFromNumber(3), writes: [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }] },
|
||||
context,
|
||||
);
|
||||
await storage.replay({ seq: seqFromNumber(7), writes: [entry(9)] }, context);
|
||||
expect(await storage.head(context)).toBe(7);
|
||||
expect((await storage.entry(entryId(9), context))?.commitSeq).toBe(7);
|
||||
expect(await storage.mintId()).toBe(10);
|
||||
expect(await storage.commit([entry(10)], context)).toBe(8);
|
||||
await expect(storage.replay({ seq: seqFromNumber(8), writes: [entry(11)] }, context)).rejects.toThrow(
|
||||
"does not strictly increase",
|
||||
);
|
||||
expect(await storage.entry(entryId(11), context)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refuses document copies in a log record", async () => {
|
||||
const { storage } = await backend.open();
|
||||
await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context);
|
||||
const copy = {
|
||||
type: "document.copy",
|
||||
record: latestDoc(3),
|
||||
source: { id: documentId(2), at: "current" },
|
||||
} as unknown as LogRecord["writes"][number];
|
||||
await expect(storage.replay({ seq: seqFromNumber(2), writes: [copy] }, context)).rejects.toThrow(
|
||||
"cannot contain a document copy",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an export that a commit interrupts", async () => {
|
||||
const { storage } = await backend.open();
|
||||
await seed(storage);
|
||||
const iterator = storage.export(context)[Symbol.asyncIterator]();
|
||||
await iterator.next();
|
||||
await storage.commit([entry(100)], context);
|
||||
await expect(async () => {
|
||||
while (!(await iterator.next()).done);
|
||||
}).rejects.toThrow("changed during export");
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
it("streams a SQLite export across pages", async () => {
|
||||
const sqlite = backends.find((backend) => backend.name === "sqlite")!;
|
||||
const { storage } = await sqlite.open();
|
||||
await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context);
|
||||
await storage.commit([create(latestDoc(2), { n: 0 })], context);
|
||||
for (let id = 3; id < 1000; id += 2) await storage.commit([entry(id), delta(2, "n", id)], context);
|
||||
const log = await collect(storage.export(context));
|
||||
// The conversation has no sequence; it joins the first record, at the document's creation.
|
||||
expect(log).toHaveLength(500);
|
||||
|
||||
const replica = new MemoryStorage();
|
||||
for (const record of log) await replica.replay(record, context);
|
||||
expect(await reads(replica)).toEqual(await reads(storage));
|
||||
});
|
||||
});
|
||||
211
packages/durable/test/sync.test.ts
Normal file
211
packages/durable/test/sync.test.ts
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { seqFromNumber } from "../src/ids.ts";
|
||||
import { MemoryStorage } from "../src/storage/memory.ts";
|
||||
import {
|
||||
compactRecords,
|
||||
decodeLogLines,
|
||||
encodeLogLine,
|
||||
LogFormatError,
|
||||
MemoryOutbox,
|
||||
type OutboxRecord,
|
||||
ReplicatingStorage,
|
||||
} from "../src/sync/index.ts";
|
||||
import { type LogRecord, ROOT_CONVERSATION_ID, type StorageWrite } from "../src/types.ts";
|
||||
import { collect, create, delta, documentId, entry, latestDoc, reads, seed, task } from "./replay-support.ts";
|
||||
|
||||
const context = BACKGROUND_CONTEXT;
|
||||
|
||||
async function replayed(records: readonly LogRecord[]): Promise<MemoryStorage> {
|
||||
const storage = new MemoryStorage();
|
||||
for (const record of records) await storage.replay(record, context);
|
||||
return storage;
|
||||
}
|
||||
|
||||
async function replicated(): Promise<{ readonly storage: ReplicatingStorage; readonly outbox: MemoryOutbox }> {
|
||||
const outbox = new MemoryOutbox();
|
||||
const storage = new ReplicatingStorage(new MemoryStorage(), outbox);
|
||||
await seed(storage);
|
||||
return { storage, outbox };
|
||||
}
|
||||
|
||||
describe("ReplicatingStorage", () => {
|
||||
it("logs every commit so that replaying the log gives the same reads", async () => {
|
||||
const { storage, outbox } = await replicated();
|
||||
expect(outbox.records.map((record) => record.seq)).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
|
||||
expect(await reads(await replayed(outbox.records))).toEqual(await reads(storage));
|
||||
});
|
||||
|
||||
it("logs fork copies as the creations they became", async () => {
|
||||
const { outbox } = await replicated();
|
||||
const fork = outbox.records.find((record) => record.seq === 6)!;
|
||||
const writes = fork.writes as readonly StorageWrite[];
|
||||
expect(writes.some((write) => write.type === "document.copy")).toBe(false);
|
||||
// The rewindable source as of seq 3, and the latest one as it was when the fork committed.
|
||||
expect(writes.filter((write) => write.type === "document.create")).toMatchObject([
|
||||
{ record: { id: 12 }, content: { kind: "base", version: 1, value: { a: 1, b: 1 } } },
|
||||
{ record: { id: 13 }, content: { kind: "base", version: 1, value: { x: 1 } } },
|
||||
]);
|
||||
});
|
||||
|
||||
it("marks changes of current-only documents created earlier", async () => {
|
||||
const { outbox } = await replicated();
|
||||
const hints = Object.fromEntries(outbox.records.map((record) => [record.seq, record.currentOnly ?? []]));
|
||||
expect(hints).toMatchObject({ 3: [6], 4: [6], 7: [10], 8: [9, 10], 9: [13] });
|
||||
expect(hints[2]).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not log replayed records", async () => {
|
||||
const outbox = new MemoryOutbox();
|
||||
const storage = new ReplicatingStorage(new MemoryStorage(), outbox);
|
||||
await storage.replay(
|
||||
{ seq: seqFromNumber(1), writes: [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }] },
|
||||
context,
|
||||
);
|
||||
expect(outbox.records).toEqual([]);
|
||||
expect(await storage.commit([entry(2)], context)).toBe(2);
|
||||
expect(outbox.records.map((record) => record.seq)).toEqual([2]);
|
||||
});
|
||||
|
||||
it("appends after the commit, so a rejected commit is never logged", async () => {
|
||||
const outbox = new MemoryOutbox();
|
||||
const storage = new ReplicatingStorage(new MemoryStorage(), outbox);
|
||||
await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context);
|
||||
await expect(
|
||||
storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context),
|
||||
).rejects.toThrow("already belongs");
|
||||
expect(outbox.records).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("tells a listener about each record once it is in the outbox", async () => {
|
||||
const outbox = new MemoryOutbox();
|
||||
const seen: number[] = [];
|
||||
const storage = new ReplicatingStorage(new MemoryStorage(), outbox, {
|
||||
onRecord: (record) => seen.push(record.seq, outbox.records.length),
|
||||
});
|
||||
await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context);
|
||||
expect(seen).toEqual([1, 1]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("compactRecords", () => {
|
||||
it("keeps the reads of every split of the log into batches", async () => {
|
||||
const { storage, outbox } = await replicated();
|
||||
const expected = await reads(storage);
|
||||
const records = outbox.records;
|
||||
for (let cut = 0; cut <= records.length; cut++) {
|
||||
const batches = [records.slice(0, cut), records.slice(cut)];
|
||||
const compacted = batches.flatMap((batch) => compactRecords(batch));
|
||||
expect(await reads(await replayed(compacted)), `cut at ${cut}`).toEqual(expected);
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the reads when every record is its own batch, or the whole log is one", async () => {
|
||||
const { storage, outbox } = await replicated();
|
||||
const expected = await reads(storage);
|
||||
expect(await reads(await replayed(outbox.records.flatMap((record) => compactRecords([record]))))).toEqual(
|
||||
expected,
|
||||
);
|
||||
expect(await reads(await replayed(compactRecords(outbox.records)))).toEqual(expected);
|
||||
});
|
||||
|
||||
it("compacts the export of a storage too", async () => {
|
||||
const { storage } = await replicated();
|
||||
const exported = await collect(storage.export(context));
|
||||
expect(await reads(await replayed(compactRecords(exported)))).toEqual(await reads(storage));
|
||||
});
|
||||
|
||||
it("drops superseded task records and streaming deltas", async () => {
|
||||
const live = latestDoc(2);
|
||||
const records: OutboxRecord[] = [
|
||||
{
|
||||
seq: seqFromNumber(10),
|
||||
writes: [{ type: "task", value: task(5, "running") }, delta(2, "partial", "a")],
|
||||
currentOnly: [documentId(2)],
|
||||
},
|
||||
{ seq: seqFromNumber(11), writes: [delta(2, "partial", "ab")], currentOnly: [documentId(2)] },
|
||||
{ seq: seqFromNumber(12), writes: [entry(6)] },
|
||||
{
|
||||
seq: seqFromNumber(13),
|
||||
writes: [
|
||||
{ type: "task", value: task(5, "terminal") },
|
||||
{ type: "document.change", id: documentId(2), content: { kind: "base", version: 1, value: {} } },
|
||||
],
|
||||
currentOnly: [documentId(2)],
|
||||
},
|
||||
{ seq: seqFromNumber(14), writes: [delta(2, "partial", "c")], currentOnly: [documentId(2)] },
|
||||
];
|
||||
expect(compactRecords(records)).toEqual([
|
||||
{ seq: 12, writes: [entry(6)] },
|
||||
{
|
||||
seq: 13,
|
||||
writes: [
|
||||
{ type: "task", value: task(5, "terminal") },
|
||||
{ type: "document.change", id: documentId(2), content: { kind: "base", version: 1, value: {} } },
|
||||
],
|
||||
currentOnly: [documentId(2)],
|
||||
},
|
||||
{ seq: 14, writes: [delta(2, "partial", "c")], currentOnly: [documentId(2)] },
|
||||
]);
|
||||
// Without the hint the document's history is unknown, so its deltas stay.
|
||||
const unhinted = records.map(({ currentOnly: _currentOnly, ...record }) => record);
|
||||
expect(compactRecords(unhinted).flatMap((record) => record.writes)).toHaveLength(6);
|
||||
expect(live.history).toBe("latest");
|
||||
});
|
||||
|
||||
it("keeps an empty last record so the batch ends where the run ended", () => {
|
||||
const records: OutboxRecord[] = [
|
||||
{ seq: seqFromNumber(3), writes: [{ type: "task", value: task(5, "pending") }] },
|
||||
{ seq: seqFromNumber(4), writes: [{ type: "task", value: task(5, "running") }] },
|
||||
{ seq: seqFromNumber(5), writes: [{ type: "task", value: task(5, "terminal") }] },
|
||||
{
|
||||
seq: seqFromNumber(6),
|
||||
writes: [create(latestDoc(7), { gone: true }), { type: "document.retire", id: documentId(7) }],
|
||||
},
|
||||
{ seq: seqFromNumber(7), writes: [{ type: "task", value: task(8, "pending") }] },
|
||||
{ seq: seqFromNumber(8), writes: [{ type: "task", value: task(8, "running") }] },
|
||||
];
|
||||
expect(compactRecords(records)).toEqual([
|
||||
{ seq: 5, writes: [{ type: "task", value: task(5, "terminal") }] },
|
||||
{
|
||||
seq: 6,
|
||||
writes: [
|
||||
{ type: "document.create", record: latestDoc(7), content: { kind: "base", version: 1, value: {} } },
|
||||
{ type: "document.retire", id: documentId(7) },
|
||||
],
|
||||
},
|
||||
{ seq: 8, writes: [{ type: "task", value: task(8, "running") }] },
|
||||
]);
|
||||
expect(compactRecords([records[0]!, { seq: seqFromNumber(4), writes: [] }])).toEqual([
|
||||
{ seq: 3, writes: [{ type: "task", value: task(5, "pending") }] },
|
||||
{ seq: 4, writes: [] },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("log lines", () => {
|
||||
it("round-trips records with their hints", async () => {
|
||||
const { outbox } = await replicated();
|
||||
const text = outbox.records.map(encodeLogLine).join("");
|
||||
expect(decodeLogLines(text)).toEqual({ records: outbox.records, torn: 0 });
|
||||
});
|
||||
|
||||
it("reports a torn last line instead of failing", () => {
|
||||
const line = encodeLogLine({ seq: seqFromNumber(1), writes: [entry(2)] });
|
||||
const torn = encodeLogLine({ seq: seqFromNumber(2), writes: [entry(3)] }).slice(0, 10);
|
||||
expect(decodeLogLines(line + torn)).toEqual({ records: [{ seq: 1, writes: [entry(2)] }], torn: 10 });
|
||||
});
|
||||
|
||||
it("refuses unknown versions, unknown writes, and sequences that do not increase", () => {
|
||||
expect(() => decodeLogLines('{"v":2,"seq":1,"writes":[]}\n')).toThrow(LogFormatError);
|
||||
expect(() => decodeLogLines('{"v":1,"seq":1,"writes":[{"type":"document.copy"}]}\n')).toThrow(
|
||||
"unknown type document.copy",
|
||||
);
|
||||
expect(() => decodeLogLines('{"v":1,"seq":2,"writes":[]}\n{"v":1,"seq":2,"writes":[]}\n')).toThrow(
|
||||
"does not increase",
|
||||
);
|
||||
expect(() => decodeLogLines("not json\n")).toThrow("is not JSON");
|
||||
// The parser's own error says where and why.
|
||||
expect(() => decodeLogLines("not json\n")).toThrow(expect.objectContaining({ cause: expect.any(SyntaxError) }));
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue