Merge branch 'main' into codemode-and-mcp

# Conflicts:
#	packages/coding-agent/install-lock/package-lock.json
#	packages/coding-agent/npm-shrinkwrap.json
This commit is contained in:
Armin Ronacher 2026-09-27 12:26:43 +02:00
commit 6e6c5a9fe3
91 changed files with 5918 additions and 2353 deletions

40
package-lock.json generated
View file

@ -4284,6 +4284,7 @@
"version": "6.40.0",
"resolved": "https://registry.npmjs.org/openai/-/openai-6.40.0.tgz",
"integrity": "sha512-MWtTjd/gQt4jpbji61NTgFWJLoY/PdRJ6wG9/ZDRMYNMlBKrCrSlkLI+KgHP1vR1qT6LKSAyAqIxno6lcK9JiA==",
"dev": true,
"license": "Apache-2.0",
"peerDependencies": {
"ws": "^8.18.0",
@ -5556,7 +5557,7 @@
"@smithy/node-http-handler": "4.12.1",
"http-proxy-agent": "9.1.0",
"https-proxy-agent": "9.1.0",
"openai": "6.40.0",
"openai": "7.19.0",
"partial-json": "0.1.7",
"typebox": "1.3.27"
},
@ -5595,6 +5596,43 @@
"node": ">= 20"
}
},
"packages/ai/node_modules/openai": {
"version": "7.19.0",
"resolved": "https://registry.npmjs.org/openai/-/openai-7.19.0.tgz",
"integrity": "sha512-MX2s3u2L5racTO0CC/SWpCOasJQBCJrqLKXK+l82cAhdeF8mPMBEe/gxMm0ZFa2xpKpOFLRjxv5afYEZbBXmbQ==",
"license": "Apache-2.0",
"engines": {
"node": ">=22.0.0"
},
"peerDependencies": {
"@aws-sdk/credential-provider-node": ">=3.972.0 <4",
"@smithy/hash-node": ">=4.3.0 <5",
"@smithy/signature-v4": ">=5.4.0 <6",
"undici": ">=5 <9",
"ws": "^8.21.0",
"zod": "^3.25 || ^4.0"
},
"peerDependenciesMeta": {
"@aws-sdk/credential-provider-node": {
"optional": true
},
"@smithy/hash-node": {
"optional": true
},
"@smithy/signature-v4": {
"optional": true
},
"undici": {
"optional": true
},
"ws": {
"optional": true
},
"zod": {
"optional": true
}
}
},
"packages/chord": {
"name": "@earendil-works/chord",
"version": "0.87.1",

View file

@ -23,6 +23,9 @@
### Fixed
- Fixed 1-hour Anthropic cache writes reported by Vercel AI Gateway in streaming deltas being priced at the 5-minute rate ([#9210](https://github.com/earendil-works/pi/issues/9210)).
- Fixed model-level `samplingParams` being dropped by direct `stream()`/`complete()` calls on OpenAI-compatible APIs ([#9506](https://github.com/earendil-works/pi/issues/9506)).
- Fixed Mistral GLM models producing empty text blocks and split thinking blocks from empty content deltas, which could make later requests fail with "Expected at most one leading ThinkChunk" ([#9674](https://github.com/earendil-works/pi/issues/9674)).
- Fixed OpenAI Fast mode requests being priced at the standard rate when the response reports `service_tier: "fast"`, as GPT-6 models do ([#10034](https://github.com/earendil-works/pi/issues/10034)).
## [0.87.1] - 2026-09-22

View file

@ -70,7 +70,7 @@
"@smithy/node-http-handler": "4.12.1",
"http-proxy-agent": "9.1.0",
"https-proxy-agent": "9.1.0",
"openai": "6.40.0",
"openai": "7.19.0",
"partial-json": "0.1.7",
"typebox": "1.3.27"
},

View file

@ -344,10 +344,8 @@ function buildParams(
}
}
// Last so custom keys override the named request fields.
if (options?.samplingParams) {
Object.assign(params, options.samplingParams);
}
// Last so custom keys override the named request fields. Per-request keys override model defaults.
Object.assign(params, model.samplingParams, options?.samplingParams);
return params;
}

View file

@ -626,6 +626,9 @@ async function consumeChatStream(
for (const item of contentItems) {
if (typeof item === "string") {
const textDelta = sanitizeSurrogates(item);
// GLM models on Mistral send empty content deltas around thinking and tool calls.
// Opening a block for them splits thinking into multiple blocks, which Mistral rejects on replay.
if (!textDelta) continue;
if (!currentBlock || currentBlock.type !== "text") {
finishCurrentBlock(currentBlock);
currentBlock = { type: "text", text: "" };
@ -667,6 +670,7 @@ async function consumeChatStream(
if (item.type === "text") {
const textDelta = sanitizeSurrogates(item.text ?? "");
if (!textDelta) continue;
if (!currentBlock || currentBlock.type !== "text") {
finishCurrentBlock(currentBlock);
currentBlock = { type: "text", text: "" };

View file

@ -827,7 +827,7 @@ function buildParams(
};
if (compat.supportsUsageInStreaming !== false) {
(params as any).stream_options = { include_usage: true };
params.stream_options = { include_usage: true };
}
if (compat.supportsStore) {
@ -836,7 +836,8 @@ function buildParams(
if (options?.maxTokens) {
if (compat.maxTokensField === "max_tokens") {
(params as any).max_tokens = options.maxTokens;
// Deprecated by OpenAI, but some OpenAI-compatible providers only accept max_tokens.
(params as { max_tokens?: number }).max_tokens = options.maxTokens;
} else {
params.max_completion_tokens = options.maxTokens;
}
@ -994,10 +995,8 @@ function buildParams(
}
}
// Last so custom keys override the named request fields.
if (options?.samplingParams) {
Object.assign(params, options.samplingParams);
}
// Last so custom keys override the named request fields. Per-request keys override model defaults.
Object.assign(params, model.samplingParams, options?.samplingParams);
return params;
}

View file

@ -92,7 +92,7 @@ function getPromptCacheRetention(
function getPromptCacheOptions(
compat: Required<OpenAIResponsesCompat>,
cacheRetention: CacheRetention,
): { mode?: "explicit"; ttl?: "30m" } | undefined {
): ResponseCreateParamsStreaming["prompt_cache_options"] {
if (!compat.supportsExplicitPromptCacheMode) return undefined;
if (cacheRetention === "none") return { mode: "explicit" };
if (cacheRetention === "long" && compat.supportsLongCacheRetention) return { ttl: "30m" };
@ -307,9 +307,7 @@ function buildParams(
});
const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env);
const params: ResponseCreateParamsStreaming & {
prompt_cache_options?: { mode?: "explicit"; ttl?: "30m" };
} = {
const params: ResponseCreateParamsStreaming = {
model: model.id,
input: messages,
stream: true,
@ -360,10 +358,8 @@ function buildParams(
if (model.provider === "xai") params.include = ["reasoning.encrypted_content"];
}
// Last so custom keys override the named request fields.
if (options?.samplingParams) {
Object.assign(params, options.samplingParams);
}
// Last so custom keys override the named request fields. Per-request keys override model defaults.
Object.assign(params, model.samplingParams, options?.samplingParams);
return params;
}
@ -376,6 +372,7 @@ function getServiceTierCostMultiplier(
case "flex":
return 0.5;
case "priority":
case "fast":
return model.id === "gpt-5.5" ? 2.5 : 2;
default:
return 1;

View file

@ -24,13 +24,9 @@ export function buildBaseOptions(
options?: SimpleStreamOptions,
apiKey?: string,
): StreamOptions {
const samplingParams =
model.samplingParams || options?.samplingParams
? { ...model.samplingParams, ...options?.samplingParams }
: undefined;
return {
temperature: options?.temperature,
samplingParams,
samplingParams: options?.samplingParams,
maxTokens: clampMaxTokensToContext(model, context, options?.maxTokens ?? model.maxTokens),
signal: options?.signal,
telemetryContext: options?.telemetryContext,

View file

@ -282,7 +282,7 @@ describe("Anthropic raw SSE parsing", () => {
} as unknown as Anthropic;
await streamAnthropic(
getModel("openrouter", "anthropic/claude-3-haiku"),
getModel("openrouter", "anthropic/claude-haiku-4.5"),
normalizeContext({ messages: [{ role: "user", content: "Hello", timestamp: 1 }] }),
{ client, thinkingEnabled: false },
).result();

View file

@ -322,6 +322,48 @@ describe("Mistral HTTP transport", () => {
expect(message.usage).toMatchObject({ input: 7, output: 4, cacheRead: 3, cacheWrite: 0, totalTokens: 14 });
});
// #9674: GLM models on Mistral send empty content deltas at the start, around tool calls,
// and sometimes mid-thinking. They must not open blocks or split thinking.
it("ignores empty content deltas", async () => {
const model = getModel("mistral", "zai-glm-5-3");
const context = normalizeContext({
messages: [{ role: "user", content: "hello", timestamp: 1 }],
});
const thinking = (text: string) => ({ type: "thinking", thinking: [{ type: "text", text }] });
const toolCall = (args: string, first: boolean) => ({
index: 0,
...(first ? { id: "abc123456" } : {}),
function: { name: first ? "read" : "", arguments: args },
});
const deltas = [
{ content: "" },
{ content: [thinking("first part,")] },
{ content: "" },
{ content: [{ type: "text", text: "" }] },
{ content: [thinking(" second part."), { type: "text", text: "Reading." }] },
{ content: "", tool_calls: [toolCall("", true)] },
{ content: "", tool_calls: [toolCall('{"path":', false)] },
{ content: "", tool_calls: [toolCall('"a.txt"}', false)] },
{ content: "" },
];
const events = deltas.map((delta, i) => ({
id: "response-1",
model: model.id,
choices: [{ index: 0, finish_reason: i === deltas.length - 1 ? "tool_calls" : null, delta }],
}));
const message = await streamMistral(model, context, {
apiKey: "test",
fetch: async () => createSseResponse(events),
}).result();
expect(message.content).toEqual([
{ type: "thinking", thinking: "first part, second part." },
{ type: "text", text: "Reading." },
{ type: "toolCall", id: "abc123456", name: "read", arguments: { path: "a.txt" } },
]);
});
it("forwards each parsed SSE payload before normalizing it", async () => {
const model = getModel("mistral", "mistral-large-latest");
const context = normalizeContext({

View file

@ -478,51 +478,60 @@ describe("openai-responses provider defaults", () => {
});
it.each([
["gpt-5.4", "priority", 2],
["gpt-5.5", "priority", 2.5],
["gpt-5.5", "flex", 0.5],
] as const)("applies %s %s service-tier cost multiplier", async (modelId, serviceTier, multiplier) => {
const model = getModel("openai", modelId);
const tokenCount = 100_000;
const tokenScale = tokenCount / 1_000_000;
const sse = `${[
`data: ${JSON.stringify({
type: "response.completed",
response: {
status: "completed",
service_tier: serviceTier,
usage: {
input_tokens: tokenCount,
output_tokens: tokenCount,
total_tokens: tokenCount * 2,
input_tokens_details: { cached_tokens: 0 },
["gpt-5.4", "priority", "priority", 2],
["gpt-5.5", "priority", "priority", 2.5],
["gpt-5.5", "flex", "flex", 0.5],
// GPT-6 models report Fast mode as "fast" even when "priority" is requested (#10034)
["gpt-6-luna", "priority", "fast", 2],
["gpt-6-luna", "fast", "fast", 2],
] as const)(
"applies %s cost multiplier for requested %s and returned %s service tier",
async (modelId, serviceTier, responseServiceTier, multiplier) => {
const model = getModel("openai", modelId);
const tokenCount = 100_000;
const tokenScale = tokenCount / 1_000_000;
const sse = `${[
`data: ${JSON.stringify({
type: "response.completed",
response: {
status: "completed",
service_tier: responseServiceTier,
usage: {
input_tokens: tokenCount,
output_tokens: tokenCount,
total_tokens: tokenCount * 2,
input_tokens_details: { cached_tokens: 0 },
},
},
},
})}`,
].join("\n\n")}\n\n`;
})}`,
].join("\n\n")}\n\n`;
vi.spyOn(globalThis, "fetch").mockResolvedValue(
new Response(sse, {
status: 200,
headers: { "content-type": "text/event-stream" },
}),
);
vi.spyOn(globalThis, "fetch").mockResolvedValue(
new Response(sse, {
status: 200,
headers: { "content-type": "text/event-stream" },
}),
);
const stream = streamOpenAIResponses(
model,
normalizeContext({
systemPrompt: "sys",
messages: [{ role: "user", content: "hi", timestamp: Date.now() }],
}),
{ apiKey: "test-key", serviceTier },
);
const stream = streamOpenAIResponses(
model,
normalizeContext({
systemPrompt: "sys",
messages: [{ role: "user", content: "hi", timestamp: Date.now() }],
}),
{ apiKey: "test-key", serviceTier },
);
const result = await stream.result();
const result = await stream.result();
expect(result.usage.cost.input).toBe(model.cost.input * multiplier * tokenScale);
expect(result.usage.cost.output).toBe(model.cost.output * multiplier * tokenScale);
expect(result.usage.cost.total).toBe((model.cost.input + model.cost.output) * multiplier * tokenScale);
});
expect(result.usage.cost.input).toBeCloseTo(model.cost.input * multiplier * tokenScale, 12);
expect(result.usage.cost.output).toBeCloseTo(model.cost.output * multiplier * tokenScale, 12);
expect(result.usage.cost.total).toBeCloseTo(
(model.cost.input + model.cost.output) * multiplier * tokenScale,
12,
);
},
);
});
describe("openai-responses max_output_tokens compat", () => {

View file

@ -1,6 +1,6 @@
import { describe, expect, it } from "vitest";
import { streamSimple } from "../src/compat.ts";
import type { Api, Context, Model, SimpleStreamOptions } from "../src/types.ts";
import { stream, streamSimple } from "../src/compat.ts";
import type { Api, Context, Model, StreamOptions } from "../src/types.ts";
interface SamplingPayload {
temperature?: number;
@ -22,11 +22,11 @@ function makeContext(): Context {
};
}
function makeCompletionsModel(overrides?: Partial<Model<"openai-completions">>): Model<"openai-completions"> {
function makeModel(api: Api, samplingParams?: Record<string, unknown>): Model<Api> {
return {
id: "custom-model",
name: "Custom Model",
api: "openai-completions",
api,
provider: "custom-provider",
baseUrl: "http://127.0.0.1:9/v1",
reasoning: false,
@ -34,38 +34,29 @@ function makeCompletionsModel(overrides?: Partial<Model<"openai-completions">>):
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 128000,
maxTokens: 16384,
...overrides,
samplingParams,
};
}
function makeAnthropicModel(): Model<"anthropic-messages"> {
function capturingOptions(onCapture: (payload: SamplingPayload) => void) {
return {
id: "vendor--claude",
name: "Vendor Proxy Claude",
api: "anthropic-messages",
provider: "vendor-proxy",
baseUrl: "http://127.0.0.1:9",
reasoning: true,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 200000,
maxTokens: 32000,
};
}
async function capturePayload(model: Model<Api>, options?: SimpleStreamOptions): Promise<SamplingPayload> {
let capturedPayload: SamplingPayload | undefined;
const s = streamSimple(model, makeContext(), {
...options,
apiKey: "fake-key",
onPayload: (payload) => {
capturedPayload = payload as SamplingPayload;
onPayload: (payload: unknown) => {
onCapture(payload as SamplingPayload);
throw new PayloadCaptured();
},
});
};
}
await s.result();
async function capturePayload(model: Model<Api>, options?: StreamOptions): Promise<SamplingPayload> {
let capturedPayload: SamplingPayload | undefined;
await stream(model, makeContext(), {
...options,
...capturingOptions((payload) => {
capturedPayload = payload;
}),
}).result();
if (!capturedPayload) {
throw new Error("Expected payload to be captured before request failure");
@ -75,8 +66,8 @@ async function capturePayload(model: Model<Api>, options?: SimpleStreamOptions):
}
describe("sampling params", () => {
it("merges stream-option sampling params into the request body", async () => {
const payload = await capturePayload(makeCompletionsModel(), {
it("merges request sampling params into the request body", async () => {
const payload = await capturePayload(makeModel("openai-completions"), {
samplingParams: { top_p: 0.95, top_k: 0, min_p: 0 },
});
@ -86,30 +77,40 @@ describe("sampling params", () => {
});
it("omits sampling params when neither options nor model set them", async () => {
const payload = await capturePayload(makeCompletionsModel());
const payload = await capturePayload(makeModel("openai-completions"));
expect(payload.temperature).toBeUndefined();
expect(payload.top_p).toBeUndefined();
});
it("applies model-level sampling params", async () => {
const payload = await capturePayload(makeCompletionsModel({ samplingParams: { temperature: 1, top_p: 0.95 } }));
// Model defaults must apply to direct stream()/complete() calls, not only streamSimple() (#9506)
it.each(["openai-completions", "openai-responses", "azure-openai-responses"] as const)(
"applies model-level sampling params with request keys taking precedence for %s",
async (api) => {
const payload = await capturePayload(makeModel(api, { top_p: 0.95, min_p: 0.05 }), {
samplingParams: { top_p: 0.5 },
});
expect(payload.temperature).toBe(1);
expect(payload.top_p).toBe(0.95);
});
expect(payload.top_p).toBe(0.5);
expect(payload.min_p).toBe(0.05);
},
);
it("merges stream-option keys over model-level keys", async () => {
const payload = await capturePayload(makeCompletionsModel({ samplingParams: { top_p: 0.95, min_p: 0.05 } }), {
it("passes request sampling params through streamSimple", async () => {
let payload: SamplingPayload | undefined;
await streamSimple(makeModel("openai-completions"), makeContext(), {
samplingParams: { top_p: 0.5 },
});
...capturingOptions((captured) => {
payload = captured;
}),
}).result();
expect(payload.top_p).toBe(0.5);
expect(payload.min_p).toBe(0.05);
expect(payload?.top_p).toBe(0.5);
});
it("overrides named request fields", async () => {
const payload = await capturePayload(makeCompletionsModel(), {
const payload = await capturePayload(makeModel("openai-completions"), {
temperature: 0,
samplingParams: { temperature: 1 },
});
@ -118,7 +119,7 @@ describe("sampling params", () => {
});
it("is ignored by non-OpenAI-compatible APIs", async () => {
const payload = await capturePayload(makeAnthropicModel(), {
const payload = await capturePayload(makeModel("anthropic-messages"), {
samplingParams: { top_p: 0.9, top_k: 40 },
});

View file

@ -14,6 +14,7 @@
### Changed
- Switched the build from the TypeScript native preview to TypeScript 7.0 with an ES2024 target, and replaced `tsx` with Node's built-in type stripping for running from source ([#9965](https://github.com/earendil-works/pi/issues/9965)).
- Removed the `[Themes]` section from the startup banner. Custom themes remain available in `/settings`, and theme conflicts are still reported.
### Fixed
@ -22,6 +23,7 @@
- Fixed pinned git extensions loaded with `-e` continuing to use the first downloaded commit after the ref changes ([#9982](https://github.com/earendil-works/pi/issues/9982)).
- Fixed `RpcClient` skipping the next event listener when a listener unsubscribes while handling an event, which could make `waitForIdle()` time out after `collectEvents()` ([#9990](https://github.com/earendil-works/pi/issues/9990)).
- Fixed full-file `read` calls rendering as `:1` when models send `null` for omitted `offset` and `limit` ([#9996](https://github.com/earendil-works/pi/issues/9996)).
- Fixed new sessions being lost when pi exits before the first assistant response. The session file is now created when the first user message is sent ([#10000](https://github.com/earendil-works/pi/issues/10000)).
## [0.87.1] - 2026-09-22

View file

@ -73,7 +73,7 @@ See [Compaction Reference](compaction.md) for trigger, summarization, and valida
| Setting | Type | Default | Description |
|---|---|---|---|
| `theme` | string | Detected | Built-in or custom theme name. |
| `theme` | string | `"system"` | Built-in or custom theme name. `system` derives colors from the terminal theme. |
| `quietStartup` | boolean | `false` | Hide the startup header. |
| `tuiMode` | `"regular" \| "fullscreen"` | `"regular"` | Interactive terminal UI mode. |
| `fullscreenExitOutput` | `"transcript" \| "resume-hint"` | `"transcript"` | Output printed when fullscreen mode exits. |

View file

@ -1,6 +1,25 @@
# Customize Pi with themes
Themes control the colors Pi uses in interactive mode and HTML exports. Pi includes `dark` and `light` themes. You can select one theme, follow your terminal's light or dark appearance, or create your own palette.
Themes control the colors Pi uses in interactive mode and HTML exports. Pi includes the `system`, `dark`, and `light` themes. You can select one theme, follow your terminal's light or dark appearance, or create your own palette.
## Use your terminal's colors
The `system` theme is the default. It builds Pi's colors from your terminal's theme, so Pi matches the terminal instead of bringing its own palette:
- Pi queries the terminal's default foreground and background colors and its 16 ANSI colors.
- Each Pi color takes its hue from one ANSI color, for example errors from red and links from blue.
- Pi sets each color's lightness so that it stands out from the background by a minimum contrast. Body text keeps at least a 4.5:1 WCAG contrast ratio on the background and every panel.
- When the terminal switches between light and dark, Pi queries the colors again and rebuilds the theme.
The theme adapts to what the terminal reports:
| Terminal reports | Result |
|---|---|
| Background and ANSI colors | Colors from the terminal palette, placed for the actual background. |
| Background only | Pi's own hues, placed for the actual background. |
| Nothing | ANSI color indices and the terminal's default colors, which the terminal renders itself. Secondary text is faint, and panels have no background color. |
Pi asks the terminal for its colors when it starts. Terminals usually answer within a few milliseconds, and Pi waits at most 100 ms before showing the startup header. If the terminal does not answer in time, Pi uses the ANSI color fallback, and it still applies the colors if they arrive later, for example over a slow SSH connection. `system` is a reserved name: a custom theme with that name is ignored.
<a id="selecting-a-theme"></a>
@ -16,6 +35,8 @@ The selection is saved as the `theme` [setting](settings.md#terminal-and-display
}
```
Without a `theme` setting, Pi uses `system`.
Automatic mode stores the light theme first and the dark theme second:
```json
@ -24,7 +45,7 @@ Automatic mode stores the light theme first and the dark theme second:
}
```
When automatic mode is active, Pi changes themes when the terminal reports an appearance change. Theme names cannot contain `/` because Pi reserves it for this setting format.
Pi decides whether the terminal is light or dark from its reported background and foreground colors. If the terminal does not report its background, Pi uses the terminal's light/dark notification, then the `COLORFGBG` environment variable, then dark. The same decision picks the theme of a light/dark pair and the appearance of `system`. When automatic mode is active, Pi changes themes when the terminal reports an appearance change. Theme names cannot contain `/` because Pi reserves it for this setting format.
Use `--use-theme` to choose the initial theme for one invocation without changing the saved setting:
@ -37,7 +58,7 @@ See [CLI resources](cli.md#resources) for the command-line option.
## Create a custom theme
Copy one of the [built-in themes](https://github.com/earendil-works/pi/tree/main/packages/coding-agent/src/modes/interactive/theme) or create a new JSON file conforming to the [schema](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/modes/interactive/theme/theme-schema.json).
Copy one of the [built-in themes](https://github.com/earendil-works/pi/tree/main/packages/coding-agent/src/modes/interactive/theme) or create a new JSON file conforming to the [schema](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/modes/interactive/theme/theme-schema.json). The built-in themes use OKHSL colors, with variables for colors that several roles share, so you can adjust a hue, saturation, or lightness directly.
1. Save the file as `<agent-dir>/themes/my-theme.json`. The agent directory defaults to `~/.pi/agent`.
2. Set its `name` to `my-theme`.
@ -51,25 +72,26 @@ Use the theme name as the filename. Pi hot-reloads the active user theme only fr
| Property | Required | Responsibility |
|---|---|---|
| `$schema` | No | Enables editor validation and completion against Pi's published schema. |
| `name` | Yes | Identifies the theme in selectors and settings. It must be unique and cannot contain `/`. |
| `name` | Yes | Identifies the theme in selectors and settings. It must be unique, cannot contain `/`, and cannot be `system`. |
| `appearance` | No | `"dark"` or `"light"`: the background the theme is designed for. Pi detects it from the theme colors when omitted. |
| `vars` | No | Defines reusable color values. Variables can reference other variables. |
| `colors` | Yes | Assigns colors to terminal UI roles. The schema identifies required and optional roles. |
| `export` | No | Overrides page and panel backgrounds in HTML exports. |
A color can be written in five forms:
A color can be written in six forms:
| Form | Example | Meaning |
|---|---|---|
| RGB hexadecimal | `"#0af"` or `"#00aaff"` | A three- or six-digit sRGB color. |
| OKLCH | `"oklch(62% 0.1 200)"` | Perceptual lightness, chroma, and hue. |
| OKHSL | `"okhsl(250 60% 55%)"` | Hue, saturation, and lightness. Saturation is relative to the most the sRGB gamut allows at that hue and lightness, so every value is in gamut and equal saturation looks equally colorful. |
| 256-color index | `39` | An ANSI palette index from `0` through `255`. |
| Variable reference | `"primary"` | The value of an entry in `vars`. |
| Terminal default | `""` | The terminal's default foreground or background color. |
Terminal default colors render as the terminal's own colors. Where Pi needs a concrete value, such as HTML export or extension color math, it uses the default colors the terminal reports, or a black or white guess based on the theme's appearance.
Pi resolves chained variable references. A missing variable or circular reference makes the theme invalid. Pi uses truecolor when available, gamut-maps OKLCH to sRGB, and approximates colors for 256-color terminals. If colors differ from their source values, check your terminal's truecolor detection and contrast settings. See [Configure Your Terminal](terminal-setup.md#override-detected-capabilities).
Pi resolves chained variable references. A missing variable or circular reference makes the theme invalid. Pi uses truecolor when available, gamut-maps OKLCH to sRGB, and approximates colors for 256-color terminals. HTML exports convert OKHSL values to hexadecimal because CSS does not support them. If colors differ from their source values, check your terminal's truecolor detection and contrast settings. See [Configure Your Terminal](terminal-setup.md#override-detected-capabilities).
Use the [theme JSON schema](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/modes/interactive/theme/theme-schema.json) for the exact properties, required colors, and accepted value types.

View file

@ -425,7 +425,7 @@
"@smithy/node-http-handler": "4.12.1",
"http-proxy-agent": "9.1.0",
"https-proxy-agent": "9.1.0",
"openai": "6.40.0",
"openai": "7.19.0",
"partial-json": "0.1.7",
"typebox": "1.3.27"
},
@ -459,6 +459,43 @@
"node": ">= 20"
}
},
"node_modules/@earendil-works/pi-ai/node_modules/openai": {
"version": "7.19.0",
"resolved": "https://registry.npmjs.org/openai/-/openai-7.19.0.tgz",
"integrity": "sha512-MX2s3u2L5racTO0CC/SWpCOasJQBCJrqLKXK+l82cAhdeF8mPMBEe/gxMm0ZFa2xpKpOFLRjxv5afYEZbBXmbQ==",
"license": "Apache-2.0",
"peerDependencies": {
"@aws-sdk/credential-provider-node": ">=3.972.0 <4",
"@smithy/hash-node": ">=4.3.0 <5",
"@smithy/signature-v4": ">=5.4.0 <6",
"undici": ">=5 <9",
"ws": "^8.21.0",
"zod": "^3.25 || ^4.0"
},
"peerDependenciesMeta": {
"@aws-sdk/credential-provider-node": {
"optional": true
},
"@smithy/hash-node": {
"optional": true
},
"@smithy/signature-v4": {
"optional": true
},
"undici": {
"optional": true
},
"ws": {
"optional": true
},
"zod": {
"optional": true
}
},
"engines": {
"node": ">=22.0.0"
}
},
"node_modules/@earendil-works/pi-codemode": {
"version": "0.87.1",
"resolved": "https://registry.npmjs.org/@earendil-works/pi-codemode/-/pi-codemode-0.87.1.tgz",
@ -1655,24 +1692,6 @@
"url": "https://opencollective.com/node-fetch"
}
},
"node_modules/openai": {
"version": "6.40.0",
"resolved": "https://registry.npmjs.org/openai/-/openai-6.40.0.tgz",
"integrity": "sha512-MWtTjd/gQt4jpbji61NTgFWJLoY/PdRJ6wG9/ZDRMYNMlBKrCrSlkLI+KgHP1vR1qT6LKSAyAqIxno6lcK9JiA==",
"license": "Apache-2.0",
"peerDependencies": {
"ws": "^8.18.0",
"zod": "^3.25 || ^4.0"
},
"peerDependenciesMeta": {
"ws": {
"optional": true
},
"zod": {
"optional": true
}
}
},
"node_modules/p-retry": {
"version": "4.6.2",
"resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz",

View file

@ -450,7 +450,7 @@
"@smithy/node-http-handler": "4.12.1",
"http-proxy-agent": "9.1.0",
"https-proxy-agent": "9.1.0",
"openai": "6.40.0",
"openai": "7.19.0",
"partial-json": "0.1.7",
"typebox": "1.3.27"
},
@ -484,6 +484,43 @@
"node": ">= 20"
}
},
"node_modules/@earendil-works/pi-ai/node_modules/openai": {
"version": "7.19.0",
"resolved": "https://registry.npmjs.org/openai/-/openai-7.19.0.tgz",
"integrity": "sha512-MX2s3u2L5racTO0CC/SWpCOasJQBCJrqLKXK+l82cAhdeF8mPMBEe/gxMm0ZFa2xpKpOFLRjxv5afYEZbBXmbQ==",
"license": "Apache-2.0",
"peerDependencies": {
"@aws-sdk/credential-provider-node": ">=3.972.0 <4",
"@smithy/hash-node": ">=4.3.0 <5",
"@smithy/signature-v4": ">=5.4.0 <6",
"undici": ">=5 <9",
"ws": "^8.21.0",
"zod": "^3.25 || ^4.0"
},
"peerDependenciesMeta": {
"@aws-sdk/credential-provider-node": {
"optional": true
},
"@smithy/hash-node": {
"optional": true
},
"@smithy/signature-v4": {
"optional": true
},
"undici": {
"optional": true
},
"ws": {
"optional": true
},
"zod": {
"optional": true
}
},
"engines": {
"node": ">=22.0.0"
}
},
"node_modules/@earendil-works/pi-codemode": {
"version": "0.87.1",
"resolved": "https://registry.npmjs.org/@earendil-works/pi-codemode/-/pi-codemode-0.87.1.tgz",
@ -1645,24 +1682,6 @@
"url": "https://opencollective.com/node-fetch"
}
},
"node_modules/openai": {
"version": "6.40.0",
"resolved": "https://registry.npmjs.org/openai/-/openai-6.40.0.tgz",
"integrity": "sha512-MWtTjd/gQt4jpbji61NTgFWJLoY/PdRJ6wG9/ZDRMYNMlBKrCrSlkLI+KgHP1vR1qT6LKSAyAqIxno6lcK9JiA==",
"license": "Apache-2.0",
"peerDependencies": {
"ws": "^8.18.0",
"zod": "^3.25 || ^4.0"
},
"peerDependenciesMeta": {
"ws": {
"optional": true
},
"zod": {
"optional": true
}
}
},
"node_modules/p-retry": {
"version": "4.6.2",
"resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz",

View file

@ -17,17 +17,19 @@ import {
FirstTimeSetupComponent,
type FirstTimeSetupResult,
} from "../modes/interactive/components/first-time-setup.ts";
import { SYSTEM_THEME_NAME } from "../modes/interactive/theme/system-theme.ts";
import {
detectTerminalBackgroundFromEnv,
detectTerminalThemeForAuto,
getTerminalTheme,
initTheme,
loadThemeFromPath,
parseAutoThemeSetting,
markTerminalColorsPending,
resolveThemeSetting,
setRegisteredThemes,
setTerminalColors,
setTheme,
type Theme,
} from "../modes/interactive/theme/theme.ts";
import { requestTerminalColors } from "../modes/interactive/theme/theme-controller.ts";
const OFFICIAL_PACKAGE_NAME = "@earendil-works/pi-coding-agent";
const OFFICIAL_APP_NAME = "pi";
@ -83,8 +85,9 @@ async function loadStartupThemes(settingsManager: SettingsManager): Promise<Them
export async function createStartupTui(settingsManager: SettingsManager): Promise<TUI> {
setCapabilityOverrides(settingsManager.getTerminalCapabilityOverrides());
setRegisteredThemes(await loadStartupThemes(settingsManager));
const terminalTheme = detectTerminalBackgroundFromEnv().theme;
initTheme(resolveThemeSetting(settingsManager.getThemeSetting(), terminalTheme) ?? terminalTheme);
// The system theme starts in grayscale until the terminal reports its colors.
markTerminalColorsPending();
initTheme(resolveThemeSetting(settingsManager.getThemeSetting(), getTerminalTheme()) ?? SYSTEM_THEME_NAME);
setKeybindings(KeybindingsManager.create());
const ui: TUI = new TuiMainScreen(new ProcessTerminal(), settingsManager.getShowHardwareCursor(), getAgentDir());
ui.setClearOnShrink(settingsManager.getClearOnShrink());
@ -93,17 +96,23 @@ export async function createStartupTui(settingsManager: SettingsManager): Promis
export function startStartupTui(ui: TUI, settingsManager: SettingsManager): void {
ui.start();
void applyDetectedStartupTheme(ui, settingsManager);
const themeSetting = settingsManager.getThemeSetting();
queryStartupTerminalColors(ui, () => {
setTheme(resolveThemeSetting(themeSetting, getTerminalTheme()) ?? SYSTEM_THEME_NAME);
});
}
async function applyDetectedStartupTheme(ui: TUI, settingsManager: SettingsManager): Promise<void> {
const themeSetting = settingsManager.getThemeSetting();
if (themeSetting && !parseAutoThemeSetting(themeSetting)) return;
const terminalTheme = await detectTerminalThemeForAuto({ ui, timeoutMs: 100 });
setTheme(resolveThemeSetting(themeSetting, terminalTheme) ?? terminalTheme);
ui.invalidate();
ui.requestRender();
/**
* Query the terminal's colors without waiting for them. When they arrive, including after the timeout,
* record them for the system theme and "" (terminal default) tokens, run `onColors`, and re-render.
*/
function queryStartupTerminalColors(ui: TUI, onColors: () => void): void {
void requestTerminalColors(ui, (colors) => {
setTerminalColors(colors);
onColors();
ui.invalidate();
ui.requestRender();
});
}
async function clearStartupTui(ui: TUI): Promise<void> {
@ -189,25 +198,23 @@ export async function showFirstTimeSetup(settingsManager: SettingsManager): Prom
resolve();
};
const showSetup = async () => {
ui.start();
const detectedTheme = await detectTerminalThemeForAuto({ ui, timeoutMs: 100 });
setTheme(detectedTheme);
const component = new FirstTimeSetupComponent({
detectedTheme,
onThemePreview: (themeName) => {
setTheme(themeName);
ui.requestRender();
},
onSubmit: (result) => void finish(result),
onCancel: () => void finish(undefined),
});
ui.addChild(component);
ui.setFocus(component);
ui.requestRender();
};
void showSetup();
ui.start();
let previewTheme = SYSTEM_THEME_NAME;
setTheme(previewTheme);
const component = new FirstTimeSetupComponent({
onThemePreview: (themeName) => {
previewTheme = themeName;
setTheme(themeName);
ui.requestRender();
},
onSubmit: (result) => void finish(result),
onCancel: () => void finish(undefined),
});
ui.addChild(component);
ui.setFocus(component);
ui.requestRender();
// The terminal's colors regenerate the system theme; re-rendering rebuilds the dialog with it.
queryStartupTerminalColors(ui, () => setTheme(previewTheme));
});
}

View file

@ -310,9 +310,7 @@ export class AgentSessionRuntime {
}
if (!existsSync(currentSessionFile)) {
throw new Error(
"This session has not been saved yet. Wait for the first assistant response before cloning or forking it.",
);
throw new Error("This session has not been saved yet. Send a message before cloning or forking it.");
}
const sessionManager = SessionManager.open(currentSessionFile, sessionDir);
const forkedSessionPath = sessionManager.createBranchedSession(targetLeafId);

View file

@ -1,5 +1,6 @@
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
import { basename, dirname, join, resolve, sep } from "node:path";
import { detectCapabilities, getTerminalColorMode, type TerminalColorMode } from "@earendil-works/pi-tui";
import chalk from "chalk";
import { CONFIG_DIR_NAME } from "../config.ts";
import { loadThemeFromPath, type Theme } from "../modes/interactive/theme/theme.ts";
@ -791,7 +792,12 @@ export class DefaultResourceLoader implements ResourceLoader {
if (this.noThemes && themePaths.length === 0) {
themesResult = { themes: [], diagnostics: [] };
} else {
const loaded = this.loadThemes(themePaths, false);
// Theme construction only needs trueColor, so skip the unrelated tmux hyperlink probe.
const colorMode = getTerminalColorMode({
...detectCapabilities(() => false),
...this.settingsManager.getTerminalCapabilityOverrides(),
});
const loaded = this.loadThemes(themePaths, false, colorMode);
const deduped = this.dedupeThemes(loaded.themes);
themesResult = { themes: deduped.themes, diagnostics: [...loaded.diagnostics, ...deduped.diagnostics] };
}
@ -934,7 +940,8 @@ export class DefaultResourceLoader implements ResourceLoader {
private loadThemes(
paths: string[],
includeDefaults: boolean = true,
includeDefaults: boolean,
colorMode: TerminalColorMode,
): {
themes: Theme[];
diagnostics: ResourceDiagnostic[];
@ -945,7 +952,7 @@ export class DefaultResourceLoader implements ResourceLoader {
const defaultDirs = [join(this.agentDir, "themes"), join(this.cwd, CONFIG_DIR_NAME, "themes")];
for (const dir of defaultDirs) {
this.loadThemesFromDir(dir, themes, diagnostics);
this.loadThemesFromDir(dir, themes, diagnostics, colorMode);
}
}
@ -959,9 +966,9 @@ export class DefaultResourceLoader implements ResourceLoader {
try {
const stats = statSync(resolved);
if (stats.isDirectory()) {
this.loadThemesFromDir(resolved, themes, diagnostics);
this.loadThemesFromDir(resolved, themes, diagnostics, colorMode);
} else if (stats.isFile() && resolved.endsWith(".json")) {
this.loadThemeFromFile(resolved, themes, diagnostics);
this.loadThemeFromFile(resolved, themes, diagnostics, colorMode);
} else {
diagnostics.push({ type: "warning", message: "theme path is not a json file", path: resolved });
}
@ -974,7 +981,12 @@ export class DefaultResourceLoader implements ResourceLoader {
return { themes, diagnostics };
}
private loadThemesFromDir(dir: string, themes: Theme[], diagnostics: ResourceDiagnostic[]): void {
private loadThemesFromDir(
dir: string,
themes: Theme[],
diagnostics: ResourceDiagnostic[],
colorMode: TerminalColorMode,
): void {
if (!existsSync(dir)) {
return;
}
@ -996,7 +1008,7 @@ export class DefaultResourceLoader implements ResourceLoader {
if (!entry.name.endsWith(".json")) {
continue;
}
this.loadThemeFromFile(join(dir, entry.name), themes, diagnostics);
this.loadThemeFromFile(join(dir, entry.name), themes, diagnostics, colorMode);
}
} catch (error) {
const message = error instanceof Error ? error.message : "failed to read theme directory";
@ -1004,9 +1016,14 @@ export class DefaultResourceLoader implements ResourceLoader {
}
}
private loadThemeFromFile(filePath: string, themes: Theme[], diagnostics: ResourceDiagnostic[]): void {
private loadThemeFromFile(
filePath: string,
themes: Theme[],
diagnostics: ResourceDiagnostic[],
colorMode: TerminalColorMode,
): void {
try {
themes.push(loadThemeFromPath(filePath));
themes.push(loadThemeFromPath(filePath, colorMode));
} catch (error) {
const message = error instanceof Error ? error.message : "failed to load theme";
diagnostics.push({ type: "warning", message, path: filePath });

View file

@ -1157,21 +1157,23 @@ export class SessionManager {
return this.sessionFile;
}
/**
* A new session file is created only once the session contains a user or assistant message.
* Setup entries alone (model, thinking level, system prompt) stay in memory so opening and
* closing pi without chatting leaves no file behind. Starting at the user message (not the
* first assistant reply) keeps the prompt on disk if the first turn never completes (#10000).
*/
private _hasConversation(): boolean {
return this.fileEntries.some(
(e) => e.type === "message" && (e.message.role === "user" || e.message.role === "assistant"),
);
}
_persist(entry: SessionEntry): void {
if (!this.persist || !this.sessionFile) return;
const hasAssistant = this.fileEntries.some((e) => e.type === "message" && e.message.role === "assistant");
if (!hasAssistant) {
if (this.flushed) {
appendFileSync(this.sessionFile, `${JSON.stringify(entry)}\n`);
} else {
// Mark as not flushed so when assistant arrives, all entries get written
this.flushed = false;
}
return;
}
if (!this.flushed) {
if (!this._hasConversation()) return;
const fd = openSync(this.sessionFile, "wx");
try {
for (const e of this.fileEntries) {
@ -1707,13 +1709,9 @@ export class SessionManager {
this.sessionFile = newSessionFile;
this._buildIndex();
// Only write the file now if it contains an assistant message.
// Otherwise defer to _persist(), which creates the file on the
// first assistant response, matching the newSession() contract
// and avoiding the duplicate-header bug when _persist()'s
// no-assistant guard later resets flushed to false.
const hasAssistant = this.fileEntries.some((e) => e.type === "message" && e.message.role === "assistant");
if (hasAssistant) {
// Use the same rule as _persist(): write now if the branched path already
// has a conversation, otherwise let _persist() create the file later.
if (this._hasConversation()) {
this._rewriteFile();
this.flushed = true;
} else {

View file

@ -783,7 +783,7 @@ export async function runClientTui(command: ClientCommand, options: RunClientTui
tui.setFocus(component);
tuiStarted = true;
tui.start();
await themeController.applyFromSettings();
themeController.applyFromSettings();
await finished;
} finally {
themeController.dispose();

View file

@ -1,22 +1,23 @@
import { Container, getKeybindings, Spacer, Text } from "@earendil-works/pi-tui";
import { APP_NAME } from "../../../config.ts";
import { type TerminalTheme, theme } from "../theme/theme.ts";
import { SYSTEM_THEME_NAME } from "../theme/system-theme.ts";
import { theme } from "../theme/theme.ts";
import { DynamicBorder } from "./dynamic-border.ts";
import { keyHint, rawKeyHint } from "./keybinding-hints.ts";
export interface FirstTimeSetupResult {
theme: TerminalTheme;
theme: string;
shareAnalytics: boolean;
}
export interface FirstTimeSetupOptions {
detectedTheme: TerminalTheme;
onThemePreview: (themeName: TerminalTheme) => void;
onThemePreview: (themeName: string) => void;
onSubmit: (result: FirstTimeSetupResult) => void;
onCancel: () => void;
}
const THEME_OPTIONS: Array<{ value: TerminalTheme; label: string }> = [
const THEME_OPTIONS: Array<{ value: string; label: string }> = [
{ value: SYSTEM_THEME_NAME, label: "System (matches your terminal colors)" },
{ value: "dark", label: "Dark" },
{ value: "light", label: "Light" },
];
@ -38,13 +39,16 @@ export class FirstTimeSetupComponent extends Container {
constructor(options: FirstTimeSetupOptions) {
super();
this.options = options;
this.themeIndex = Math.max(
0,
THEME_OPTIONS.findIndex((option) => option.value === options.detectedTheme),
);
this.themeIndex = 0;
this.update();
}
/** Rebuild on theme changes, e.g. when the system theme receives the terminal's colors. */
override invalidate(): void {
this.update();
super.invalidate();
}
// Rebuild the whole dialog on every change so theme previews recolor all text.
private update(): void {
this.clear();
@ -59,7 +63,6 @@ export class FirstTimeSetupComponent extends Container {
if (this.step === "theme") {
this.addChild(new Text(theme.fg("text", "Pick a theme."), 1, 0));
this.addChild(new Text(theme.fg("muted", `Detected system appearance: ${this.options.detectedTheme}`), 1, 0));
this.addChild(new Spacer(1));
this.addOptionList(
THEME_OPTIONS.map((option) => option.label),

View file

@ -0,0 +1,26 @@
import { backgroundAnsi, foregroundAnsi, rgbColor } from "@earendil-works/pi-tui";
import { theme } from "../theme/theme.ts";
const CORAL = rgbColor(228, 138, 122);
const BLUE = rgbColor(79, 142, 179);
const YELLOW = rgbColor(234, 182, 93);
const RESET = "\x1b[0m";
/**
* The pi logo: 4 cells wide and 2 lines tall. Each cell shows two square pixels with half blocks:
*
* coral coral coral .
* blue . coral .
* blue blue . yellow
* blue . . yellow
*
* The brand colors stay fixed across themes; they follow the terminal's color mode.
*/
export function piLogoLines(): [string, string] {
const mode = theme.getColorMode();
const fg = (color: typeof CORAL) => foregroundAnsi(color, mode);
// The fourth cell of the top line is empty, so it is padded to the same width as the bottom line.
const top = `${fg(CORAL)}${backgroundAnsi(BLUE, mode)}▀${RESET}${fg(CORAL)}▀█${RESET} `;
const bottom = `${fg(BLUE)}█▀${RESET} ${fg(YELLOW)}█${RESET}`;
return [top, bottom];
}

View file

@ -21,7 +21,13 @@ import {
type TuiMode,
type WarningSettings,
} from "../../../core/settings-manager.ts";
import { getSettingsListTheme, parseAutoThemeSetting, type TerminalTheme, theme } from "../theme/theme.ts";
import {
getSettingsListTheme,
parseAutoThemeSetting,
SYSTEM_THEME_NAME,
type TerminalTheme,
theme,
} from "../theme/theme.ts";
import { DynamicBorder } from "./dynamic-border.ts";
import { keyDisplayText } from "./keybinding-hints.ts";
import { SelectSubmenu, SteppedSubmenu, type SteppedSubmenuStep } from "./settings-submenu.ts";
@ -199,19 +205,25 @@ function themeItems(availableThemes: string[], currentTheme: string): SelectItem
return availableThemes.map((name) => ({
value: name,
label: `${name === currentTheme ? "✓ " : " "}${name}`,
...(name === SYSTEM_THEME_NAME ? { description: "Theme created from your terminal's colors" } : {}),
}));
}
const AUTOMATIC_THEME_VALUE = "/";
/** The system theme comes first, then automatic mode, then the remaining themes. */
function singleModeThemeItems(availableThemes: string[], currentTheme: string): SelectItem[] {
const items = themeItems(availableThemes, currentTheme);
const systemIndex = items.findIndex((item) => item.value === SYSTEM_THEME_NAME);
const system = systemIndex === -1 ? [] : items.splice(systemIndex, 1);
return [
...system,
{
value: AUTOMATIC_THEME_VALUE,
label: " Automatic",
label: " automatic",
description: "Use separate themes for light and dark terminal appearance",
},
...themeItems(availableThemes, currentTheme),
...items,
];
}
@ -229,7 +241,7 @@ function defaultAutomaticThemes(
if (autoTheme) return autoTheme;
const currentFixedTheme = currentThemeSetting.includes("/") ? undefined : currentThemeSetting;
const themeName = preferredTheme(availableThemes, currentFixedTheme, "dark");
const themeName = preferredTheme(availableThemes, currentFixedTheme, SYSTEM_THEME_NAME);
return { lightTheme: themeName, darkTheme: themeName };
}
@ -267,7 +279,7 @@ class ThemeSubmenu extends Container {
this.singleTheme = preferredTheme(
availableThemes,
fixedTheme ?? (autoTheme ? this.getActiveAutomaticTheme() : undefined),
"dark",
SYSTEM_THEME_NAME,
);
if (this.mode === "automatic") {
@ -291,7 +303,7 @@ class ThemeSubmenu extends Container {
this.mode = "single";
const menu = new SelectSubmenu(
"Theme",
"Select a theme, or choose Automatic to follow terminal appearance.",
"Select a theme, or choose automatic to follow terminal appearance.",
singleModeThemeItems(this.availableThemes, this.singleTheme),
this.singleTheme,
(value) => {

View file

@ -0,0 +1,32 @@
import { Text } from "@earendil-works/pi-tui";
/**
* Text whose content applies theme colors. Plain `Text` keeps the colors its string was built with, so
* a theme change, or the system theme receiving the terminal's colors, would leave it stale. This
* rebuilds the string from `build` after every invalidation, which the UI performs on theme changes.
*
* `build` must return the same content each time, apart from colors. Snapshot changing data before
* creating the component, or call `invalidate()` after changing state that `build` reads.
*/
export class ThemedText extends Text {
private readonly build: () => string;
private stale = true;
constructor(build: () => string, paddingX = 1, paddingY = 1) {
super("", paddingX, paddingY);
this.build = build;
}
override invalidate(): void {
super.invalidate();
this.stale = true;
}
override render(width: number): string[] {
if (this.stale) {
this.stale = false;
this.setText(this.build());
}
return super.render(width);
}
}

View file

@ -152,6 +152,7 @@ import {
formatAuthSelectorProviderType,
OAuthSelectorComponent,
} from "./components/oauth-selector.ts";
import { piLogoLines } from "./components/pi-logo.ts";
import { ScopedModelsSelectorComponent } from "./components/scoped-models-selector.ts";
import { SessionSelectorComponent } from "./components/session-selector.ts";
import { SettingsSelectorComponent } from "./components/settings-selector.ts";
@ -164,6 +165,7 @@ import {
type StatusIndicator,
WorkingStatusIndicator,
} from "./components/status-indicator.ts";
import { ThemedText } from "./components/themed-text.ts";
import { ThinkingSelectorComponent } from "./components/thinking-selector.ts";
import { ToolExecutionComponent } from "./components/tool-execution.ts";
import { TreeSelectorComponent } from "./components/tree-selector.ts";
@ -181,6 +183,7 @@ import {
getMarkdownTheme,
getThemeByName,
onThemeChange,
SYSTEM_THEME_NAME,
setRegisteredThemes,
stopThemeWatcher,
Theme,
@ -215,9 +218,8 @@ function isExpandable(obj: unknown): obj is Expandable {
return typeof obj === "object" && obj !== null && "setExpanded" in obj && typeof obj.setExpanded === "function";
}
class ExpandableText extends Text implements Expandable {
private readonly getCollapsedText: () => string;
private readonly getExpandedText: () => string;
class ExpandableText extends ThemedText implements Expandable {
private readonly state: { expanded: boolean };
constructor(
getCollapsedText: () => string,
@ -226,13 +228,14 @@ class ExpandableText extends Text implements Expandable {
paddingX = 0,
paddingY = 0,
) {
super(expanded ? getExpandedText() : getCollapsedText(), paddingX, paddingY);
this.getCollapsedText = getCollapsedText;
this.getExpandedText = getExpandedText;
const state = { expanded };
super(() => (state.expanded ? getExpandedText() : getCollapsedText()), paddingX, paddingY);
this.state = state;
}
setExpanded(expanded: boolean): void {
this.setText(expanded ? this.getExpandedText() : this.getCollapsedText());
this.state.expanded = expanded;
this.invalidate();
}
}
@ -460,7 +463,8 @@ export class InteractiveMode {
// Status line tracking (for mutating immediately-sequential status updates)
private lastStatusSpacer: Spacer | undefined = undefined;
private lastStatusText: Text | undefined = undefined;
private lastStatusText: ThemedText | undefined = undefined;
private lastStatusMessage = "";
private managedToolStatusStarted = false;
// Streaming message tracking
@ -572,7 +576,7 @@ export class InteractiveMode {
});
this.runtimeHost.setRebindSession(async () => {
await this.rebindCurrentSession({ renderBeforeBind: true });
await this.themeController.applyFromSettings();
this.themeController.applyFromSettings();
});
this.version = VERSION;
this.renderer = createInteractiveTui({
@ -814,7 +818,7 @@ export class InteractiveMode {
const condensedText = `Updated to v${latestVersion}. Use ${theme.bold("/changelog")} to view full changelog.`;
this.chatContainer.addChild(new Text(condensedText, 1, 0));
} else {
this.chatContainer.addChild(new Text(theme.bold(theme.fg("accent", "What's New")), 1, 0));
this.chatContainer.addChild(new ThemedText(() => theme.bold(theme.fg("accent", "What's New")), 1, 0));
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(
new Markdown(this.changelogMarkdown.trim(), 1, 0, this.getMarkdownThemeWithSettings()),
@ -951,54 +955,63 @@ export class InteractiveMode {
this.ui.start();
this.isInitialized = true;
await this.themeController.applyFromSettings();
this.themeController.applyFromSettings();
// The header and startup notices bake theme colors into their text, so build them once the terminal
// reported its colors. This ends at the terminal's DA1 reply, or after 100 ms if it answers nothing.
await this.themeController.waitForTerminalColors();
// Add header with keybindings from config (unless silenced)
if (this.options.verbose || !this.settingsManager.getQuietStartup()) {
const logo = theme.bold(theme.fg("accent", APP_NAME)) + theme.fg("dim", ` v${this.version}`);
// Built on demand so the header follows theme changes. The logo's first line carries the version,
// its second line the first line of key hints.
const withLogo = (hints: string) => {
const [top, bottom] = piLogoLines();
return `${top} ${theme.fg("dim", `v${this.version}`)}\n${bottom} ${hints}`;
};
// Build startup instructions using keybinding hint helpers
const hint = (keybinding: AppKeybinding, description: string) => keyHint(keybinding, description);
const expandedInstructions = [
hint("app.interrupt", "to interrupt"),
hint("app.clear", "to clear"),
rawKeyHint(`${keyText("app.clear")} twice`, "to exit"),
hint("app.exit", "to exit (empty)"),
hint("app.suspend", "to suspend"),
keyHint("tui.editor.deleteToLineEnd", "to delete to end"),
hint("app.thinking.cycle", "to cycle thinking level"),
rawKeyHint(`${keyText("app.model.cycleForward")}/${keyText("app.model.cycleBackward")}`, "to cycle models"),
hint("app.model.select", "to select model"),
hint("app.tools.expand", "to expand tools"),
hint("app.thinking.toggle", "to expand thinking"),
hint("app.editor.external", "for external editor"),
rawKeyHint("/", "for commands"),
rawKeyHint("!", "to run bash"),
rawKeyHint("!!", "to run bash (no context)"),
hint("app.message.followUp", "to queue follow-up"),
hint("app.message.dequeue", "to edit all queued messages"),
hint("app.clipboard.pasteImage", "to paste image (with text fallback)"),
rawKeyHint("drop files", "to attach"),
].join("\n");
const compactInstructions = [
hint("app.interrupt", "interrupt"),
rawKeyHint(`${keyText("app.clear")}/${keyText("app.exit")}`, "clear/exit"),
rawKeyHint("/", "commands"),
rawKeyHint("!", "bash"),
hint("app.tools.expand", "more"),
].join(theme.fg("muted", " · "));
const compactOnboarding = theme.fg(
"dim",
`Press ${keyText("app.tools.expand")} to show full startup help and loaded resources.`,
);
const onboarding = theme.fg(
"dim",
`Pi can explain its own features and look up its docs. Ask it how to use or extend Pi.`,
);
const expandedInstructions = () =>
[
hint("app.interrupt", "to interrupt"),
hint("app.clear", "to clear"),
rawKeyHint(`${keyText("app.clear")} twice`, "to exit"),
hint("app.exit", "to exit (empty)"),
hint("app.suspend", "to suspend"),
keyHint("tui.editor.deleteToLineEnd", "to delete to end"),
hint("app.thinking.cycle", "to cycle thinking level"),
rawKeyHint(
`${keyText("app.model.cycleForward")}/${keyText("app.model.cycleBackward")}`,
"to cycle models",
),
hint("app.model.select", "to select model"),
hint("app.tools.expand", "to expand tools"),
hint("app.thinking.toggle", "to expand thinking"),
hint("app.editor.external", "for external editor"),
rawKeyHint("/", "for commands"),
rawKeyHint("!", "to run bash"),
rawKeyHint("!!", "to run bash (no context)"),
hint("app.message.followUp", "to queue follow-up"),
hint("app.message.dequeue", "to edit all queued messages"),
hint("app.clipboard.pasteImage", "to paste image (with text fallback)"),
rawKeyHint("drop files", "to attach"),
].join("\n");
const compactInstructions = () =>
[
hint("app.interrupt", "interrupt"),
rawKeyHint(`${keyText("app.clear")}/${keyText("app.exit")}`, "clear/exit"),
rawKeyHint("/", "commands"),
rawKeyHint("!", "bash"),
hint("app.tools.expand", "more"),
].join(theme.fg("muted", " · "));
const compactOnboarding = () =>
theme.fg("dim", `Press ${keyText("app.tools.expand")} to show full startup help and loaded resources.`);
const onboarding = () =>
theme.fg("dim", `Pi can explain its own features and look up its docs. Ask it how to use or extend Pi.`);
this.builtInHeader = new ExpandableText(
() => `${logo}\n${compactInstructions}\n${compactOnboarding}\n\n${onboarding}`,
() => `${logo}\n${expandedInstructions}\n\n${onboarding}`,
() => `${withLogo(compactInstructions())}\n${compactOnboarding()}\n\n${onboarding()}`,
() => `${withLogo(expandedInstructions())}\n\n${onboarding()}`,
this.getStartupExpansionState(),
1,
0,
@ -1714,15 +1727,16 @@ export class InteractiveMode {
}
return theme.fg("dim", ` ${labels.join(", ")}`);
};
// Bodies are built on demand so the listing follows theme changes.
const addLoadedSection = (
name: string,
collapsedBody: string,
collapsedBody: () => string,
expandedBody = collapsedBody,
color: ThemeColor = "mdHeading",
): void => {
const section = new ExpandableText(
() => `${sectionHeader(name, color)}\n${collapsedBody}`,
() => `${sectionHeader(name, color)}\n${expandedBody}`,
() => `${sectionHeader(name, color)}\n${collapsedBody()}`,
() => `${sectionHeader(name, color)}\n${expandedBody()}`,
this.getStartupExpansionState(),
0,
0,
@ -1774,13 +1788,13 @@ export class InteractiveMode {
];
if (contextFiles.length > 0) {
this.loadedResourcesContainer.addChild(new Spacer(1));
const contextList = contextFiles
.map((f) => theme.fg("dim", ` ${this.formatDisplayPath(f.path)}`))
.join("\n");
const contextCompactList = formatCompactList(
contextFiles.map((contextFile) => this.formatContextPath(contextFile.path)),
{ sort: false },
);
const contextList = () =>
contextFiles.map((f) => theme.fg("dim", ` ${this.formatDisplayPath(f.path)}`)).join("\n");
const contextCompactList = () =>
formatCompactList(
contextFiles.map((contextFile) => this.formatContextPath(contextFile.path)),
{ sort: false },
);
addLoadedSection("Context", contextCompactList, contextList);
}
@ -1789,11 +1803,12 @@ export class InteractiveMode {
const groups = this.buildScopeGroups(
skills.map((skill) => ({ path: skill.filePath, sourceInfo: skill.sourceInfo })),
);
const skillList = this.formatScopeGroups(groups, {
formatPath: (item) => this.formatDisplayPath(item.path),
formatPackagePath: (item) => this.getShortPath(item.path, item.sourceInfo),
});
const skillCompactList = formatCompactList(skills.map((skill) => skill.name));
const skillList = () =>
this.formatScopeGroups(groups, {
formatPath: (item) => this.formatDisplayPath(item.path),
formatPackagePath: (item) => this.getShortPath(item.path, item.sourceInfo),
});
const skillCompactList = () => formatCompactList(skills.map((skill) => skill.name));
addLoadedSection("Skills", skillCompactList, skillList);
}
@ -1803,70 +1818,50 @@ export class InteractiveMode {
templates.map((template) => ({ path: template.filePath, sourceInfo: template.sourceInfo })),
);
const templateByPath = new Map(templates.map((t) => [t.filePath, t]));
const templateList = this.formatScopeGroups(groups, {
formatPath: (item) => {
const template = templateByPath.get(item.path);
return template ? `/${template.name}` : this.formatDisplayPath(item.path);
},
formatPackagePath: (item) => {
const template = templateByPath.get(item.path);
return template ? `/${template.name}` : this.formatDisplayPath(item.path);
},
});
const promptCompactList = formatCompactList(templates.map((template) => `/${template.name}`));
const templateList = () =>
this.formatScopeGroups(groups, {
formatPath: (item) => {
const template = templateByPath.get(item.path);
return template ? `/${template.name}` : this.formatDisplayPath(item.path);
},
formatPackagePath: (item) => {
const template = templateByPath.get(item.path);
return template ? `/${template.name}` : this.formatDisplayPath(item.path);
},
});
const promptCompactList = () => formatCompactList(templates.map((template) => `/${template.name}`));
addLoadedSection("Prompts", promptCompactList, templateList);
}
if (extensions.length > 0) {
const groups = this.buildScopeGroups(extensions);
const extList = this.formatScopeGroups(groups, {
formatPath: (item) => this.formatExtensionDisplayPath(item.path),
formatPackagePath: (item) =>
this.formatExtensionDisplayPath(this.getShortPath(item.path, item.sourceInfo)),
});
const extensionCompactList = formatCompactList(this.getCompactExtensionLabels(extensions));
const extList = () =>
this.formatScopeGroups(groups, {
formatPath: (item) => this.formatExtensionDisplayPath(item.path),
formatPackagePath: (item) =>
this.formatExtensionDisplayPath(this.getShortPath(item.path, item.sourceInfo)),
});
const extensionLabels = this.getCompactExtensionLabels(extensions);
const extensionCompactList = () => formatCompactList(extensionLabels);
addLoadedSection("Extensions", extensionCompactList, extList, "mdHeading");
}
// Show loaded themes (excluding built-in)
const loadedThemes = themesResult.themes;
const customThemes = loadedThemes.filter((t) => t.sourcePath);
if (customThemes.length > 0) {
const groups = this.buildScopeGroups(
customThemes.map((loadedTheme) => ({
path: loadedTheme.sourcePath!,
sourceInfo: loadedTheme.sourceInfo,
})),
);
const themeList = this.formatScopeGroups(groups, {
formatPath: (item) => this.formatDisplayPath(item.path),
formatPackagePath: (item) => this.getShortPath(item.path, item.sourceInfo),
});
const themeCompactList = formatCompactList(
customThemes.map(
(loadedTheme) =>
loadedTheme.name ?? this.getCompactPathLabel(loadedTheme.sourcePath!, loadedTheme.sourceInfo),
),
);
addLoadedSection("Themes", themeCompactList, themeList);
}
}
if (showDiagnostics) {
const skillDiagnostics = skillsResult.diagnostics;
if (skillDiagnostics.length > 0) {
const warningLines = this.formatDiagnostics(skillDiagnostics, sourceInfos);
const warningLines = () => this.formatDiagnostics(skillDiagnostics, sourceInfos);
this.loadedResourcesContainer.addChild(
new Text(`${theme.fg("warning", "[Skill conflicts]")}\n${warningLines}`, 0, 0),
new ThemedText(() => `${theme.fg("warning", "[Skill conflicts]")}\n${warningLines()}`, 0, 0),
);
this.loadedResourcesContainer.addChild(new Spacer(1));
}
const promptDiagnostics = promptsResult.diagnostics;
if (promptDiagnostics.length > 0) {
const warningLines = this.formatDiagnostics(promptDiagnostics, sourceInfos);
const warningLines = () => this.formatDiagnostics(promptDiagnostics, sourceInfos);
this.loadedResourcesContainer.addChild(
new Text(`${theme.fg("warning", "[Prompt conflicts]")}\n${warningLines}`, 0, 0),
new ThemedText(() => `${theme.fg("warning", "[Prompt conflicts]")}\n${warningLines()}`, 0, 0),
);
this.loadedResourcesContainer.addChild(new Spacer(1));
}
@ -1888,18 +1883,18 @@ export class InteractiveMode {
extensionDiagnostics.push(...shortcutDiagnostics);
if (extensionDiagnostics.length > 0) {
const warningLines = this.formatDiagnostics(extensionDiagnostics, sourceInfos);
const warningLines = () => this.formatDiagnostics(extensionDiagnostics, sourceInfos);
this.loadedResourcesContainer.addChild(
new Text(`${theme.fg("warning", "[Extension issues]")}\n${warningLines}`, 0, 0),
new ThemedText(() => `${theme.fg("warning", "[Extension issues]")}\n${warningLines()}`, 0, 0),
);
this.loadedResourcesContainer.addChild(new Spacer(1));
}
const themeDiagnostics = themesResult.diagnostics;
if (themeDiagnostics.length > 0) {
const warningLines = this.formatDiagnostics(themeDiagnostics, sourceInfos);
const warningLines = () => this.formatDiagnostics(themeDiagnostics, sourceInfos);
this.loadedResourcesContainer.addChild(
new Text(`${theme.fg("warning", "[Theme conflicts]")}\n${warningLines}`, 0, 0),
new ThemedText(() => `${theme.fg("warning", "[Theme conflicts]")}\n${warningLines()}`, 0, 0),
);
this.loadedResourcesContainer.addChild(new Spacer(1));
}
@ -2050,10 +2045,11 @@ export class InteractiveMode {
this.showError(`${prefix}: ${message}`);
const extensionHint = this.getCrashExtensionHint(error);
if (extensionHint) {
this.chatContainer.addChild(new Text(theme.fg("warning", extensionHint), this.outputPad, 0));
this.chatContainer.addChild(new ThemedText(() => theme.fg("warning", extensionHint), this.outputPad, 0));
}
if (this.recordCrash("fatal_error", error)) {
this.chatContainer.addChild(new Text(theme.fg("muted", this.crashReportInstructions()), this.outputPad, 0));
const instructions = this.crashReportInstructions();
this.chatContainer.addChild(new ThemedText(() => theme.fg("muted", instructions), this.outputPad, 0));
}
stopThemeWatcher();
this.stop("transcript");
@ -2098,8 +2094,8 @@ export class InteractiveMode {
if (this.bugReportHintShown) return;
this.bugReportHintShown = true;
this.chatContainer.addChild(
new Text(
theme.fg("muted", `If this looks like a ${APP_NAME} bug, /bug sends a report to the developers.`),
new ThemedText(
() => theme.fg("muted", `If this looks like a ${APP_NAME} bug, /bug sends a report to the developers.`),
this.outputPad,
0,
),
@ -2328,7 +2324,7 @@ export class InteractiveMode {
container.addChild(new Text(line, 1, 0));
}
if (content.length > InteractiveMode.MAX_WIDGET_LINES) {
container.addChild(new Text(theme.fg("muted", "... (widget truncated)"), 1, 0));
container.addChild(new ThemedText(() => theme.fg("muted", "... (widget truncated)"), 1, 0));
}
component = container;
} else {
@ -2940,17 +2936,14 @@ export class InteractiveMode {
*/
private showExtensionError(extensionPath: string, error: string, stack?: string): void {
const errorMsg = `Extension "${extensionPath}" error: ${error}`;
const errorText = new Text(theme.fg("error", errorMsg), 1, 0);
const errorText = new ThemedText(() => theme.fg("error", errorMsg), 1, 0);
this.chatContainer.addChild(errorText);
if (stack) {
// Show stack trace in dim color, indented
const stackLines = stack
.split("\n")
.slice(1) // Skip first line (duplicates error message)
.map((line) => theme.fg("dim", ` ${line.trim()}`))
.join("\n");
if (stackLines) {
this.chatContainer.addChild(new Text(stackLines, 1, 0));
const stackLines = stack.split("\n").slice(1); // Skip first line (duplicates error message)
if (stackLines.length > 0) {
const renderStack = () => stackLines.map((line) => theme.fg("dim", ` ${line.trim()}`)).join("\n");
this.chatContainer.addChild(new ThemedText(renderStack, 1, 0));
}
}
this.ui.requestRender();
@ -3599,7 +3592,8 @@ export class InteractiveMode {
this.showError(event.errorMessage);
} else {
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new Text(theme.fg("error", event.errorMessage), 1, 0));
const errorMessage = event.errorMessage;
this.chatContainer.addChild(new ThemedText(() => theme.fg("error", errorMessage), 1, 0));
}
}
void this.flushCompactionQueue({ willRetry: event.willRetry });
@ -3681,7 +3675,7 @@ export class InteractiveMode {
}
const message = status.type === "warning" ? `Warning: ${status.message}` : status.message;
const color = status.type === "warning" ? "warning" : "dim";
this.chatContainer.addChild(new Text(theme.fg(color, message), 1, 0));
this.chatContainer.addChild(new ThemedText(() => theme.fg(color, message), 1, 0));
this.lastStatusSpacer = undefined;
this.lastStatusText = undefined;
this.ui.requestRender();
@ -3699,13 +3693,15 @@ export class InteractiveMode {
const secondLast = children.length > 1 ? children[children.length - 2] : undefined;
if (last && secondLast && last === this.lastStatusText && secondLast === this.lastStatusSpacer) {
this.lastStatusText.setText(theme.fg("dim", message));
this.lastStatusMessage = message;
this.lastStatusText.invalidate();
this.ui.requestRender();
return;
}
const spacer = new Spacer(1);
const text = new Text(theme.fg("dim", message), 1, 0);
this.lastStatusMessage = message;
const text = new ThemedText(() => theme.fg("dim", this.lastStatusMessage), 1, 0);
this.chatContainer.addChild(spacer);
this.chatContainer.addChild(text);
this.lastStatusSpacer = spacer;
@ -3963,7 +3959,8 @@ export class InteractiveMode {
private addCacheWarmingUsage(entry: UsageEntry): void {
if (!this.settingsManager.getShowCacheMissNotices()) return;
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new Text(theme.fg("dim", formatCacheWarmingUsage(entry)), 1, 0));
const usage = formatCacheWarmingUsage(entry);
this.chatContainer.addChild(new ThemedText(() => theme.fg("dim", usage), 1, 0));
}
/**
@ -3979,7 +3976,7 @@ export class InteractiveMode {
const label = notice.kind === "compaction" ? "Compaction" : "Branch summary";
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(
new Text(theme.fg("warning", `${label}: ${formatTokens(tokens)} tokens billed${cost}`), 1, 0),
new ThemedText(() => theme.fg("warning", `${label}: ${formatTokens(tokens)} tokens billed${cost}`), 1, 0),
);
}
@ -4021,7 +4018,11 @@ export class InteractiveMode {
const noun = droppedCount === 1 ? "thinking block" : "thinking blocks";
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(
new Text(theme.fg("warning", `Anthropic dropped ${droppedCount} ${noun} (details in session)`), 1, 0),
new ThemedText(
() => theme.fg("warning", `Anthropic dropped ${droppedCount} ${noun} (details in session)`),
1,
0,
),
);
}
@ -4049,9 +4050,8 @@ export class InteractiveMode {
} else if (miss.idleMs >= CACHE_TTL_MS) {
label = `Cache miss after ${Math.round(miss.idleMs / 60_000)}m idle`;
}
const text = theme.fg("warning", `${label}: ${reBilled}`);
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new Text(text, 1, 0));
this.chatContainer.addChild(new ThemedText(() => theme.fg("warning", `${label}: ${reBilled}`), 1, 0));
}
renderInitialMessages(): void {
@ -4080,11 +4080,12 @@ export class InteractiveMode {
this.chatContainer.addChild(new Spacer(1));
}
this.chatContainer.addChild(
new Text(
theme.fg(
"warning",
`This project is not trusted. Project ${CONFIG_DIR_NAME} resources and packages are ignored. Use /trust to save a trust decision, then restart pi.`,
),
new ThemedText(
() =>
theme.fg(
"warning",
`This project is not trusted. Project ${CONFIG_DIR_NAME} resources and packages are ignored. Use /trust to save a trust decision, then restart pi.`,
),
1,
0,
),
@ -4463,30 +4464,33 @@ export class InteractiveMode {
showError(errorMessage: string): void {
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new Text(theme.fg("error", `Error: ${errorMessage}`), this.outputPad, 0));
this.chatContainer.addChild(new ThemedText(() => theme.fg("error", `Error: ${errorMessage}`), this.outputPad, 0));
this.ui.requestRender();
}
showWarning(warningMessage: string): void {
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new Text(theme.fg("warning", `Warning: ${warningMessage}`), 1, 0));
this.chatContainer.addChild(new ThemedText(() => theme.fg("warning", `Warning: ${warningMessage}`), 1, 0));
this.ui.requestRender();
}
showNewVersionNotification(release: LatestPiRelease): void {
const action = theme.fg("accent", `${APP_NAME} update`);
const updateInstruction = theme.fg("muted", `New version ${release.version} is available. Run `) + action;
const updateInstruction = () =>
theme.fg("muted", `New version ${release.version} is available. Run `) +
theme.fg("accent", `${APP_NAME} update`);
const changelogUrl = "https://pi.dev/changelog";
const changelogLink = getCapabilities().hyperlinks
? hyperlink(theme.fg("accent", changelogUrl), changelogUrl)
: theme.fg("accent", changelogUrl);
const changelogLine = theme.fg("muted", "Changelog: ") + changelogLink;
const changelogLine = () => {
const changelogLink = getCapabilities().hyperlinks
? hyperlink(theme.fg("accent", changelogUrl), changelogUrl)
: theme.fg("accent", changelogUrl);
return theme.fg("muted", "Changelog: ") + changelogLink;
};
const note = release.note?.trim();
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new DynamicBorder((text) => theme.fg("warning", text)));
this.chatContainer.addChild(
new Text(`${theme.bold(theme.fg("warning", "Update Available"))}\n${updateInstruction}`, 1, 0),
new ThemedText(() => `${theme.bold(theme.fg("warning", "Update Available"))}\n${updateInstruction()}`, 1, 0),
);
if (note) {
this.chatContainer.addChild(new Spacer(1));
@ -4497,21 +4501,23 @@ export class InteractiveMode {
);
this.chatContainer.addChild(new Spacer(1));
}
this.chatContainer.addChild(new Text(changelogLine, 1, 0));
this.chatContainer.addChild(new ThemedText(changelogLine, 1, 0));
this.chatContainer.addChild(new DynamicBorder((text) => theme.fg("warning", text)));
this.ui.requestRender();
}
showPackageUpdateNotification(packages: string[]): void {
const action = theme.fg("accent", `${APP_NAME} update --extensions`);
const updateInstruction = theme.fg("muted", "Package updates are available. Run ") + action;
const updateInstruction = () =>
theme.fg("muted", "Package updates are available. Run ") +
theme.fg("accent", `${APP_NAME} update --extensions`);
const packageLines = packages.map((pkg) => `- ${pkg}`).join("\n");
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new DynamicBorder((text) => theme.fg("warning", text)));
this.chatContainer.addChild(
new Text(
`${theme.bold(theme.fg("warning", "Package Updates Available"))}\n${updateInstruction}\n${theme.fg("muted", "Packages:")}\n${packageLines}`,
new ThemedText(
() =>
`${theme.bold(theme.fg("warning", "Package Updates Available"))}\n${updateInstruction()}\n${theme.fg("muted", "Packages:")}\n${packageLines}`,
1,
0,
),
@ -4767,7 +4773,7 @@ export class InteractiveMode {
thinkingLevel: this.settingsManager.getDefaultThinkingLevel() ?? DEFAULT_THINKING_LEVEL,
availableThinkingLevels: [...THINKING_LEVEL_OPTIONS],
modelThinkingLevels: this.settingsManager.getAllModelThinkingLevels(),
currentTheme: this.themeController.getThemeSelection() || "dark",
currentTheme: this.themeController.getThemeSelection() || SYSTEM_THEME_NAME,
terminalTheme: this.themeController.getTerminalTheme(),
availableThemes: getAvailableThemes(),
hideThinkingBlock: this.hideThinkingBlock,
@ -4864,7 +4870,7 @@ export class InteractiveMode {
},
onThemeChange: (themeSetting) => {
this.settingsManager.setTheme(themeSetting);
void this.themeController.setThemeSetting(themeSetting);
this.themeController.setThemeSetting(themeSetting);
},
onThemePreview: (themeName) => this.themeController.preview(themeName),
onHideThinkingBlockChange: (hidden) => {
@ -6185,8 +6191,8 @@ export class InteractiveMode {
reloadBox.addChild(new DynamicBorder(borderColor));
reloadBox.addChild(new Spacer(1));
reloadBox.addChild(
new Text(
theme.fg("muted", "Reloading keybindings, extensions, skills, prompts, themes, and context files..."),
new ThemedText(
() => theme.fg("muted", "Reloading keybindings, extensions, skills, prompts, themes, and context files..."),
1,
0,
),
@ -6230,7 +6236,7 @@ export class InteractiveMode {
}
setRegisteredThemes(this.session.resourceLoader.getThemes().themes);
this.applyRuntimeSettings();
await this.themeController.applyFromSettings();
this.themeController.applyFromSettings();
this.setupAutocompleteProvider();
const runner = this.session.extensionRunner;
this.setupExtensionShortcuts(runner);
@ -6412,7 +6418,7 @@ export class InteractiveMode {
const currentName = this.sessionManager.getSessionName();
if (currentName) {
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new Text(theme.fg("dim", `Session name: ${currentName}`), 1, 0));
this.chatContainer.addChild(new ThemedText(() => theme.fg("dim", `Session name: ${currentName}`), 1, 0));
} else {
this.showWarning("Usage: /name <name>");
}
@ -6426,7 +6432,8 @@ export class InteractiveMode {
this.showWarning(`Session name was normalized from ${JSON.stringify(name)} to ${JSON.stringify(sessionName)}`);
}
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new Text(theme.fg("dim", `Session name set: ${sessionName ?? name}`), 1, 0));
const displayName = sessionName ?? name;
this.chatContainer.addChild(new ThemedText(() => theme.fg("dim", `Session name set: ${displayName}`), 1, 0));
this.ui.requestRender();
}
@ -6441,65 +6448,70 @@ export class InteractiveMode {
// grouped separately so the breakdown reconciles with the session total.
const usageBreakdown = getUsageCostBreakdown(entries);
let info = `${theme.bold("Session Info")}\n\n`;
if (sessionName) {
info += `${theme.fg("dim", "Name:")} ${sessionName}\n`;
}
info += `${theme.fg("dim", "File:")} ${stats.sessionFile ?? "In-memory"}\n`;
info += `${theme.fg("dim", "ID:")} ${stats.sessionId}\n\n`;
info += `${theme.bold("Messages")}\n`;
info += `${theme.fg("dim", "Total:")} ${stats.totalMessages}\n`;
info += `${theme.fg("dim", "User:")} ${stats.userMessages}\n`;
info += `${theme.fg("dim", "Assistant:")} ${stats.assistantMessages}\n`;
info += `${theme.fg("dim", "Tools:")} ${stats.toolCalls} calls, ${stats.toolResults} results\n\n`;
info += `${theme.bold("Tokens")}\n`;
// "Input" is the full prompt volume. With cache activity, split it into
// cached (served from cache) vs uncached (everything else) - the only
// provider-independent split. Cache writes, where reported, are a detail
// of the uncached portion.
const { input, cacheRead, cacheWrite } = stats.tokens;
const promptTokens = input + cacheRead + cacheWrite;
info += `${theme.fg("dim", "Input:")} ${promptTokens.toLocaleString()}\n`;
if (promptTokens > 0 && (cacheRead > 0 || cacheWrite > 0)) {
const hitRate = theme.fg("dim", `(${((cacheRead / promptTokens) * 100).toFixed(1)}%)`);
info += ` ${theme.fg("dim", "Cached:")} ${cacheRead.toLocaleString()} ${hitRate}\n`;
const written =
cacheWrite > 0 ? ` ${theme.fg("dim", `(${cacheWrite.toLocaleString()} written to cache)`)}` : "";
info += ` ${theme.fg("dim", "Uncached:")} ${(input + cacheWrite).toLocaleString()}${written}\n`;
}
info += `${theme.fg("dim", "Output:")} ${stats.tokens.output.toLocaleString()}\n`;
info += `${theme.fg("dim", "Total:")} ${stats.tokens.total.toLocaleString()}\n`;
// Snapshot the stats; the text is built on demand so it follows theme changes.
const cacheWarmingStatus = this.session.cacheWarmingStatus;
info += `\n${theme.bold("Cache Warming")}\n`;
info += `${theme.fg("dim", "Mode:")} ${this.settingsManager.getCacheWarmingMode()}\n`;
info += `${theme.fg("dim", "Status:")} ${cacheWarmingStatus ? formatCacheWarmingStatus(cacheWarmingStatus) : "Inactive (cache warming unavailable)"}\n`;
const decision = cacheWarmingStatus?.decision;
if (decision?.economicsAvailable) {
info += `${theme.fg("dim", "Cache miss penalty:")} $${decision.missCost.toFixed(3)}\n`;
info += `${theme.fg("dim", "Refresh cost:")} $${decision.warmCost.toFixed(3)}\n`;
}
const cacheWarmingMode = this.settingsManager.getCacheWarmingMode();
const renderInfo = (): string => {
let info = `${theme.bold("Session Info")}\n\n`;
if (sessionName) {
info += `${theme.fg("dim", "Name:")} ${sessionName}\n`;
}
info += `${theme.fg("dim", "File:")} ${stats.sessionFile ?? "In-memory"}\n`;
info += `${theme.fg("dim", "ID:")} ${stats.sessionId}\n\n`;
info += `${theme.bold("Messages")}\n`;
info += `${theme.fg("dim", "Total:")} ${stats.totalMessages}\n`;
info += `${theme.fg("dim", "User:")} ${stats.userMessages}\n`;
info += `${theme.fg("dim", "Assistant:")} ${stats.assistantMessages}\n`;
info += `${theme.fg("dim", "Tools:")} ${stats.toolCalls} calls, ${stats.toolResults} results\n\n`;
info += `${theme.bold("Tokens")}\n`;
// "Input" is the full prompt volume. With cache activity, split it into
// cached (served from cache) vs uncached (everything else) - the only
// provider-independent split. Cache writes, where reported, are a detail
// of the uncached portion.
const { input, cacheRead, cacheWrite } = stats.tokens;
const promptTokens = input + cacheRead + cacheWrite;
info += `${theme.fg("dim", "Input:")} ${promptTokens.toLocaleString()}\n`;
if (promptTokens > 0 && (cacheRead > 0 || cacheWrite > 0)) {
const hitRate = theme.fg("dim", `(${((cacheRead / promptTokens) * 100).toFixed(1)}%)`);
info += ` ${theme.fg("dim", "Cached:")} ${cacheRead.toLocaleString()} ${hitRate}\n`;
const written =
cacheWrite > 0 ? ` ${theme.fg("dim", `(${cacheWrite.toLocaleString()} written to cache)`)}` : "";
info += ` ${theme.fg("dim", "Uncached:")} ${(input + cacheWrite).toLocaleString()}${written}\n`;
}
info += `${theme.fg("dim", "Output:")} ${stats.tokens.output.toLocaleString()}\n`;
info += `${theme.fg("dim", "Total:")} ${stats.tokens.total.toLocaleString()}\n`;
if (stats.cost > 0 || cacheWaste.missedTokens > 0) {
info += `\n${theme.bold("Cost")}\n`;
info += `${theme.fg("dim", "Total:")} $${stats.cost.toFixed(3)}`;
if (usageBreakdown.length > 1) {
for (const entry of usageBreakdown) {
info += `\n ${theme.fg("dim", `${entry.key}:`)} $${entry.cost.toFixed(3)} ${theme.fg("dim", `(${formatTokens(entry.tokens)} tokens)`)}`;
info += `\n${theme.bold("Cache Warming")}\n`;
info += `${theme.fg("dim", "Mode:")} ${cacheWarmingMode}\n`;
info += `${theme.fg("dim", "Status:")} ${cacheWarmingStatus ? formatCacheWarmingStatus(cacheWarmingStatus) : "Inactive (cache warming unavailable)"}\n`;
const decision = cacheWarmingStatus?.decision;
if (decision?.economicsAvailable) {
info += `${theme.fg("dim", "Cache miss penalty:")} $${decision.missCost.toFixed(3)}\n`;
info += `${theme.fg("dim", "Refresh cost:")} $${decision.warmCost.toFixed(3)}\n`;
}
if (stats.cost > 0 || cacheWaste.missedTokens > 0) {
info += `\n${theme.bold("Cost")}\n`;
info += `${theme.fg("dim", "Total:")} $${stats.cost.toFixed(3)}`;
if (usageBreakdown.length > 1) {
for (const entry of usageBreakdown) {
info += `\n ${theme.fg("dim", `${entry.key}:`)} $${entry.cost.toFixed(3)} ${theme.fg("dim", `(${formatTokens(entry.tokens)} tokens)`)}`;
}
}
if (cacheWaste.missedTokens > 0) {
const missLabel = cacheWaste.missCount === 1 ? "1 miss" : `${cacheWaste.missCount} misses`;
const detail = `${cacheWaste.missedTokens.toLocaleString()} tokens, ${missLabel}`;
info +=
cacheWaste.missedCost >= 0.0001
? `\n${theme.fg("dim", "Cache Re-billed:")} $${cacheWaste.missedCost.toFixed(3)} ${theme.fg("dim", `(${detail})`)}`
: `\n${theme.fg("dim", "Cache Re-billed:")} ${detail}`;
}
}
if (cacheWaste.missedTokens > 0) {
const missLabel = cacheWaste.missCount === 1 ? "1 miss" : `${cacheWaste.missCount} misses`;
const detail = `${cacheWaste.missedTokens.toLocaleString()} tokens, ${missLabel}`;
info +=
cacheWaste.missedCost >= 0.0001
? `\n${theme.fg("dim", "Cache Re-billed:")} $${cacheWaste.missedCost.toFixed(3)} ${theme.fg("dim", `(${detail})`)}`
: `\n${theme.fg("dim", "Cache Re-billed:")} ${detail}`;
}
}
return info;
};
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new Text(info, 1, 0));
this.chatContainer.addChild(new ThemedText(renderInfo, 1, 0));
this.ui.requestRender();
}
@ -6517,7 +6529,7 @@ export class InteractiveMode {
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new DynamicBorder());
this.chatContainer.addChild(new Text(theme.bold(theme.fg("accent", "What's New")), 1, 0));
this.chatContainer.addChild(new ThemedText(() => theme.bold(theme.fg("accent", "What's New")), 1, 0));
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new Markdown(changelogMarkdown, 1, 1, this.getMarkdownThemeWithSettings()));
this.chatContainer.addChild(new DynamicBorder());
@ -6648,7 +6660,7 @@ export class InteractiveMode {
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new DynamicBorder());
this.chatContainer.addChild(new Text(theme.bold(theme.fg("accent", "Keyboard Shortcuts")), 1, 0));
this.chatContainer.addChild(new ThemedText(() => theme.bold(theme.fg("accent", "Keyboard Shortcuts")), 1, 0));
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new Markdown(hotkeys.trim(), 1, 1, this.getMarkdownThemeWithSettings()));
this.chatContainer.addChild(new DynamicBorder());
@ -6663,7 +6675,7 @@ export class InteractiveMode {
return;
}
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new Text(`${theme.fg("accent", "✓ New session started")}`, 1, 1));
this.chatContainer.addChild(new ThemedText(() => theme.fg("accent", "✓ New session started"), 1, 1));
this.ui.requestRender();
} catch (error: unknown) {
await this.handleFatalRuntimeError("Failed to create session", error);
@ -6698,7 +6710,7 @@ export class InteractiveMode {
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(
new Text(`${theme.fg("accent", "✓ Debug log written")}\n${theme.fg("muted", debugLogPath)}`, 1, 1),
new ThemedText(() => `${theme.fg("accent", "✓ Debug log written")}\n${theme.fg("muted", debugLogPath)}`, 1, 1),
);
this.ui.requestRender();
}

View file

@ -1,91 +1,78 @@
{
"$schema": "https://raw.githubusercontent.com/earendil-works/pi/main/packages/coding-agent/src/modes/interactive/theme/theme-schema.json",
"name": "dark",
"appearance": "dark",
"vars": {
"cyan": "#00d7ff",
"blue": "#5f87ff",
"green": "#b5bd68",
"red": "#cc6666",
"yellow": "#ffff00",
"text": "#d4d4d4",
"gray": "#808080",
"dimGray": "#666666",
"darkGray": "#505050",
"accent": "#8abeb7",
"selectedBg": "#3a3a4a",
"userMsgBg": "#343541",
"toolPendingBg": "#282832",
"toolSuccessBg": "#283228",
"toolErrorBg": "#3c2828",
"customMsgBg": "#2d2838"
"text": "okhsl(234 3% 89%)",
"muted": "okhsl(229 6% 67%)",
"violet": "okhsl(295 50% 67%)",
"blue": "okhsl(232 54% 67%)",
"green": "okhsl(159 59% 67%)",
"red": "okhsl(20 72% 67%)",
"yellow": "okhsl(83 88% 67%)",
"blueBg": "okhsl(233 41% 24%)"
},
"colors": {
"accent": "accent",
"border": "blue",
"borderAccent": "cyan",
"borderMuted": "darkGray",
"accent": "violet",
"border": "okhsl(231 57% 65%)",
"borderAccent": "okhsl(295 53% 64%)",
"borderMuted": "okhsl(229 8% 53%)",
"success": "green",
"error": "red",
"warning": "yellow",
"muted": "gray",
"dim": "dimGray",
"muted": "muted",
"dim": "okhsl(229 8% 56%)",
"text": "text",
"thinkingText": "gray",
"selectedBg": "selectedBg",
"scrollbarTrack": "darkGray",
"scrollbarThumb": "text",
"searchMatchBg": "selectedBg",
"searchMatchText": "text",
"userMessageBg": "userMsgBg",
"thinkingText": "okhsl(226 7% 65%)",
"selectedBg": "blueBg",
"scrollbarTrack": "okhsl(237 7% 33%)",
"scrollbarThumb": "okhsl(232 7% 65%)",
"searchMatchBg": "okhsl(53 51% 24%)",
"searchMatchText": "muted",
"userMessageBg": "blueBg",
"userMessageText": "text",
"customMessageBg": "customMsgBg",
"customMessageText": "text",
"customMessageLabel": "#9575cd",
"toolPendingBg": "toolPendingBg",
"toolSuccessBg": "toolSuccessBg",
"toolErrorBg": "toolErrorBg",
"customMessageBg": "okhsl(295 42% 24%)",
"customMessageText": "muted",
"customMessageLabel": "violet",
"toolPendingBg": "okhsl(229 5% 24%)",
"toolSuccessBg": "okhsl(158 46% 25%)",
"toolErrorBg": "okhsl(19 54% 25%)",
"toolTitle": "text",
"toolOutput": "gray",
"mdHeading": "#f0c674",
"mdLink": "#81a2be",
"mdLinkUrl": "dimGray",
"mdCode": "accent",
"toolOutput": "muted",
"mdHeading": "yellow",
"mdLink": "blue",
"mdLinkUrl": "muted",
"mdCode": "violet",
"mdCodeBlock": "green",
"mdCodeBlockBorder": "gray",
"mdQuote": "gray",
"mdQuoteBorder": "gray",
"mdHr": "gray",
"mdListBullet": "accent",
"mdCodeBlockBorder": "muted",
"mdQuote": "muted",
"mdQuoteBorder": "muted",
"mdHr": "muted",
"mdListBullet": "violet",
"toolDiffAdded": "green",
"toolDiffRemoved": "red",
"toolDiffContext": "gray",
"syntaxComment": "#6A9955",
"syntaxKeyword": "#569CD6",
"syntaxFunction": "#DCDCAA",
"syntaxVariable": "#9CDCFE",
"syntaxString": "#CE9178",
"syntaxNumber": "#B5CEA8",
"syntaxType": "#4EC9B0",
"syntaxOperator": "#D4D4D4",
"syntaxPunctuation": "#D4D4D4",
"thinkingOff": "darkGray",
"thinkingMinimal": "#6e6e6e",
"thinkingLow": "#5f87af",
"thinkingMedium": "#81a2be",
"thinkingHigh": "#b294bb",
"thinkingXhigh": "#d183e8",
"thinkingMax": "#ff5fff",
"bashMode": "green"
"toolDiffContext": "muted",
"syntaxComment": "muted",
"syntaxKeyword": "blue",
"syntaxFunction": "yellow",
"syntaxVariable": "okhsl(202 58% 67%)",
"syntaxString": "okhsl(52 67% 67%)",
"syntaxNumber": "green",
"syntaxType": "violet",
"syntaxOperator": "muted",
"syntaxPunctuation": "muted",
"thinkingOff": "okhsl(229 8% 49%)",
"thinkingMinimal": "okhsl(232 20% 52%)",
"thinkingLow": "okhsl(232 45% 54%)",
"thinkingMedium": "okhsl(263 59% 56%)",
"thinkingHigh": "okhsl(295 73% 59%)",
"thinkingXhigh": "okhsl(337 81% 61%)",
"thinkingMax": "okhsl(20 99% 63%)",
"bashMode": "okhsl(159 64% 65%)"
},
"export": {
"pageBg": "#18181e",
"cardBg": "#1e1e24",
"infoBg": "#3c3728"
"pageBg": "okhsl(262 14% 16%)",
"cardBg": "okhsl(264 13% 19%)",
"infoBg": "okhsl(53 51% 24%)"
}
}

View file

@ -1,90 +1,78 @@
{
"$schema": "https://raw.githubusercontent.com/earendil-works/pi/main/packages/coding-agent/src/modes/interactive/theme/theme-schema.json",
"name": "light",
"appearance": "light",
"vars": {
"teal": "#5a8080",
"blue": "#547da7",
"green": "#588458",
"red": "#aa5555",
"yellow": "#9a7326",
"text": "#1f2328",
"mediumGray": "#6c6c6c",
"dimGray": "#767676",
"lightGray": "#b0b0b0",
"selectedBg": "#d0d0e0",
"userMsgBg": "#e8e8e8",
"toolPendingBg": "#e8e8f0",
"toolSuccessBg": "#e8f0e8",
"toolErrorBg": "#f0e8e8",
"customMsgBg": "#ede7f6"
"text": "okhsl(225 5% 27%)",
"muted": "okhsl(229 8% 47%)",
"violet": "okhsl(295 60% 46%)",
"blue": "okhsl(231 68% 47%)",
"green": "okhsl(159 75% 46%)",
"red": "okhsl(20 91% 47%)",
"yellow": "okhsl(83 99% 47%)",
"blueBg": "okhsl(235 19% 91%)"
},
"colors": {
"accent": "teal",
"border": "blue",
"borderAccent": "teal",
"borderMuted": "lightGray",
"accent": "violet",
"border": "okhsl(231 67% 55%)",
"borderAccent": "okhsl(295 59% 55%)",
"borderMuted": "okhsl(235 7% 66%)",
"success": "green",
"error": "red",
"warning": "yellow",
"muted": "mediumGray",
"dim": "dimGray",
"muted": "muted",
"dim": "okhsl(229 7% 59%)",
"text": "text",
"thinkingText": "mediumGray",
"selectedBg": "selectedBg",
"scrollbarTrack": "lightGray",
"scrollbarThumb": "text",
"searchMatchBg": "selectedBg",
"searchMatchText": "text",
"userMessageBg": "userMsgBg",
"thinkingText": "okhsl(234 8% 55%)",
"selectedBg": "blueBg",
"scrollbarTrack": "okhsl(248 3% 90%)",
"scrollbarThumb": "okhsl(226 7% 65%)",
"searchMatchBg": "okhsl(56 22% 91%)",
"searchMatchText": "muted",
"userMessageBg": "blueBg",
"userMessageText": "text",
"customMessageBg": "customMsgBg",
"customMessageText": "text",
"customMessageLabel": "#7e57c2",
"toolPendingBg": "toolPendingBg",
"toolSuccessBg": "toolSuccessBg",
"toolErrorBg": "toolErrorBg",
"customMessageBg": "okhsl(295 25% 91%)",
"customMessageText": "muted",
"customMessageLabel": "violet",
"toolPendingBg": "okhsl(248 3% 91%)",
"toolSuccessBg": "okhsl(156 21% 91%)",
"toolErrorBg": "okhsl(24 23% 91%)",
"toolTitle": "text",
"toolOutput": "mediumGray",
"toolOutput": "muted",
"mdHeading": "yellow",
"mdLink": "blue",
"mdLinkUrl": "dimGray",
"mdCode": "teal",
"mdLinkUrl": "muted",
"mdCode": "violet",
"mdCodeBlock": "green",
"mdCodeBlockBorder": "mediumGray",
"mdQuote": "mediumGray",
"mdQuoteBorder": "mediumGray",
"mdHr": "mediumGray",
"mdListBullet": "green",
"mdCodeBlockBorder": "muted",
"mdQuote": "muted",
"mdQuoteBorder": "muted",
"mdHr": "muted",
"mdListBullet": "violet",
"toolDiffAdded": "green",
"toolDiffRemoved": "red",
"toolDiffContext": "mediumGray",
"syntaxComment": "#008000",
"syntaxKeyword": "#0000FF",
"syntaxFunction": "#795E26",
"syntaxVariable": "#001080",
"syntaxString": "#A31515",
"syntaxNumber": "#098658",
"syntaxType": "#267F99",
"syntaxOperator": "#000000",
"syntaxPunctuation": "#000000",
"thinkingOff": "lightGray",
"thinkingMinimal": "#767676",
"thinkingLow": "blue",
"thinkingMedium": "teal",
"thinkingHigh": "#875f87",
"thinkingXhigh": "#8b008b",
"thinkingMax": "#af005f",
"bashMode": "green"
"toolDiffContext": "muted",
"syntaxComment": "muted",
"syntaxKeyword": "blue",
"syntaxFunction": "yellow",
"syntaxVariable": "okhsl(203 73% 46%)",
"syntaxString": "okhsl(52 84% 46%)",
"syntaxNumber": "green",
"syntaxType": "violet",
"syntaxOperator": "muted",
"syntaxPunctuation": "muted",
"thinkingOff": "okhsl(223 5% 80%)",
"thinkingMinimal": "okhsl(229 14% 78%)",
"thinkingLow": "okhsl(232 33% 76%)",
"thinkingMedium": "okhsl(264 48% 74%)",
"thinkingHigh": "okhsl(295 62% 72%)",
"thinkingXhigh": "okhsl(337 74% 70%)",
"thinkingMax": "okhsl(20 98% 68%)",
"bashMode": "okhsl(159 74% 55%)"
},
"export": {
"pageBg": "#f8f8f8",
"cardBg": "#ffffff",
"infoBg": "#fffae6"
"pageBg": "okhsl(17 3% 94%)",
"cardBg": "okhsl(17 5% 97%)",
"infoBg": "okhsl(56 22% 91%)"
}
}

View file

@ -0,0 +1,636 @@
/**
* The `system` theme: pi's colors derived from the terminal's own theme.
*
* Every token belongs to a color family (its hue) and has contrast rules: it must reach a contrast level
* on the background and on the panels it is drawn on. Hue and saturation come from the terminal's palette
* color for the family's ANSI slot, or from the family's own hue when the terminal reports no palette.
* Lightness comes from the rules alone. Colors are built in OKHSL, whose saturation is relative to the
* sRGB gamut, and fade toward gray near black and white.
*
* A contrast level is a target-lightness curve: the OKLab lightness a token needs, given the lightness of
* the surface below it. The curves were fitted to the reference theme design from the "Pi themes: system
* and light/dark" review. On dark backgrounds they aim for nearly fixed lightness; on light backgrounds the
* required difference grows as the background darkens.
*
* Depending on what the terminal reports, the theme is generated in one of three tiers:
* - background and palette: hues from the palette, lightness from the background;
* - background only: the families' own hues, lightness from the background;
* - nothing: ANSI palette indices and the default colors, which the terminal renders itself.
*/
import {
colorToOkhsl,
colorToOklch,
type OkhslChannels,
okhslColor,
oklabToOkhslLightness,
type RgbColor,
rgbColor,
} from "@earendil-works/pi-tui";
import type { ThemeAppearance, ThemeBg, ThemeColor, ThemeToken } from "./theme.ts";
export const SYSTEM_THEME_NAME = "system";
// ============================================================================
// Recipe: color families and their tokens
// ============================================================================
/** A family's OKHSL hue and saturation: `max` at mid lightness, falling toward `min` at black and white. */
interface Family {
hue: number;
saturation: { min: number; max: number };
/** ANSI palette slot the family takes its hue and saturation from. */
slot: number;
}
const FAMILIES = {
neutral: { hue: 231.49, saturation: { min: 0.02, max: 0.08 }, slot: 8 },
blue: { hue: 231.49, saturation: { min: 0.1, max: 0.68 }, slot: 4 },
green: { hue: 158.68, saturation: { min: 0.1, max: 0.76 }, slot: 2 },
red: { hue: 20, saturation: { min: 0.1, max: 0.92 }, slot: 1 },
yellow: { hue: 82.36, saturation: { min: 0.5, max: 1 }, slot: 3 },
orange: { hue: 52, saturation: { min: 0.12, max: 0.85 }, slot: 3 },
violet: { hue: 295, saturation: { min: 0.2, max: 0.6 }, slot: 5 },
calamine: { hue: 202.43, saturation: { min: 0.1, max: 0.74 }, slot: 6 },
thinkingSlate: { hue: 231.49, saturation: { min: 0.08, max: 0.2 }, slot: 4 },
thinkingBlue: { hue: 231.49, saturation: { min: 0.2, max: 0.45 }, slot: 4 },
thinkingPeriwinkle: { hue: 263.25, saturation: { min: 0.3, max: 0.6 }, slot: 6 },
thinkingViolet: { hue: 295, saturation: { min: 0.4, max: 0.75 }, slot: 5 },
thinkingMagenta: { hue: 337.5, saturation: { min: 0.5, max: 0.85 }, slot: 13 },
thinkingRed: { hue: 20, saturation: { min: 0.95, max: 1 }, slot: 1 },
} satisfies Record<string, Family>;
type FamilyName = keyof typeof FAMILIES;
const TOKEN_FAMILIES: Record<ThemeToken, FamilyName> = {
selectedBg: "blue",
searchMatchBg: "orange",
userMessageBg: "blue",
customMessageBg: "violet",
toolPendingBg: "neutral",
toolSuccessBg: "green",
toolErrorBg: "red",
text: "neutral",
userMessageText: "neutral",
customMessageText: "neutral",
toolTitle: "neutral",
syntaxOperator: "neutral",
syntaxPunctuation: "neutral",
muted: "neutral",
dim: "neutral",
thinkingText: "neutral",
toolOutput: "neutral",
mdLinkUrl: "neutral",
mdQuote: "neutral",
mdQuoteBorder: "neutral",
mdHr: "neutral",
mdCodeBlockBorder: "neutral",
toolDiffContext: "neutral",
syntaxComment: "neutral",
scrollbarTrack: "neutral",
scrollbarThumb: "neutral",
searchMatchText: "neutral",
borderMuted: "neutral",
accent: "violet",
borderAccent: "violet",
customMessageLabel: "violet",
mdCode: "violet",
mdListBullet: "violet",
syntaxType: "violet",
border: "blue",
mdLink: "blue",
syntaxKeyword: "blue",
syntaxVariable: "calamine",
success: "green",
mdCodeBlock: "green",
toolDiffAdded: "green",
bashMode: "green",
syntaxNumber: "green",
error: "red",
toolDiffRemoved: "red",
warning: "yellow",
mdHeading: "yellow",
syntaxFunction: "yellow",
syntaxString: "orange",
thinkingOff: "neutral",
thinkingMinimal: "thinkingSlate",
thinkingLow: "thinkingBlue",
thinkingMedium: "thinkingPeriwinkle",
thinkingHigh: "thinkingViolet",
thinkingXhigh: "thinkingMagenta",
thinkingMax: "thinkingRed",
};
/** Palette slots for tokens that would otherwise share a hue with a similar token. */
const TOKEN_SLOTS: Partial<Record<ThemeToken, number>> = { syntaxString: 2, syntaxNumber: 5, searchMatchBg: 3 };
// ============================================================================
// Contrast levels and rules
// ============================================================================
/**
* Target-lightness curves: a polynomial in the surface's OKLab lightness giving the OKLab lightness a token
* needs on it. `reachable` is the range of surface lightness where the level can be reached; beyond it the
* level is relaxed.
*/
interface Curve {
coefficients: number[];
reachable: [number, number];
}
const LEVELS = {
panel: {
dark: { coefficients: [0.29131, -0.39746, 2.33185, -0.85524, -1.2076, 0.86276], reachable: [0, 0.979] },
light: { coefficients: [-3.74073, 27.94549, -78.44258, 112.6798, -79.60015, 22.11277], reachable: [0.348, 1] },
},
track: {
dark: { coefficients: [0.39028, -0.23015, 0.83573, 2.43829, -4.38292, 2.01582], reachable: [0, 0.946] },
light: { coefficients: [-5.24921, 38.37322, -107.28833, 152.10005, -106.17127, 29.18061], reachable: [0.368, 1] },
},
thinking0: {
dark: { coefficients: [0.52988, -0.05809, -0.30924, 4.63567, -6.52933, 2.89108], reachable: [0, 0.873] },
light: {
coefficients: [-28.27749, 182.85284, -469.62416, 603.15916, -384.59976, 97.35147],
reachable: [0.51, 1],
},
},
thinking1: {
dark: { coefficients: [0.55278, -0.03667, -0.45659, 4.95347, -6.90265, 3.0706], reachable: [0, 0.858] },
light: {
coefficients: [-37.10484, 235.86282, -596.62344, 754.3633, -474.00763, 118.3551],
reachable: [0.535, 1],
},
},
thinking2: {
dark: { coefficients: [0.57486, -0.01765, -0.58987, 5.25227, -7.27175, 3.25532], reachable: [0, 0.842] },
light: {
coefficients: [-59.89653, 377.05024, -945.07843, 1182.03145, -734.96375, 181.68658],
reachable: [0.556, 1],
},
},
thinking3: {
dark: { coefficients: [0.59621, -0.00062, -0.71148, 5.53588, -7.6392, 3.44606], reachable: [0, 0.827] },
light: {
coefficients: [-72.07122, 445.84082, -1099.57352, 1353.88793, -829.53392, 202.26164],
reachable: [0.58, 1],
},
},
thinking4: {
dark: { coefficients: [0.61691, 0.01462, -0.82288, 5.80651, -8.00641, 3.64333], reachable: [0, 0.811] },
light: {
coefficients: [-110.14338, 674.21488, -1645.75941, 2004.32367, -1215.15899, 293.3183],
reachable: [0.6, 1],
},
},
thinking5: {
dark: { coefficients: [0.63702, 0.02826, -0.92498, 6.06465, -8.37246, 3.84651], reachable: [0, 0.795] },
light: {
coefficients: [-175.47701, 1063.54495, -2570.70594, 3098.80776, -1860.15527, 444.76392],
reachable: [0.62, 1],
},
},
thinking6: {
dark: { coefficients: [0.65658, 0.04044, -1.01835, 6.30989, -8.73529, 4.05439], reachable: [0, 0.779] },
light: {
coefficients: [-183.81712, 1094.70055, -2602.68539, 3088.71276, -1826.91131, 430.75931],
reachable: [0.643, 1],
},
},
subtle: {
dark: { coefficients: [0.56762, -0.02475, -0.5383, 5.12628, -7.10931, 3.17324], reachable: [0, 0.848] },
light: {
coefficients: [-232.85459, 1376.54473, -3249.11801, 3827.91186, -2248.29472, 526.55751],
reachable: [0.657, 1],
},
},
thumb: {
dark: { coefficients: [0.60323, 0.00278, -0.73328, 5.57157, -7.68067, 3.46933], reachable: [0, 0.823] },
light: {
coefficients: [-82.89897, 511.01355, -1255.98095, 1540.76821, -940.68087, 228.58523],
reachable: [0.586, 1],
},
},
readable: {
dark: { coefficients: [0.66937, 0.04704, -1.06871, 6.43941, -8.9332, 4.17229], reachable: [0, 0.77] },
light: {
coefficients: [-1554.52576, 8733.56817, -19604.93507, 21977.72696, -12300.99599, 2749.81288],
reachable: [0.751, 1],
},
},
emphasis: {
dark: { coefficients: [0.7303, 0.07695, -1.31626, 7.1681, -10.14436, 4.92846], reachable: [0, 0.712] },
light: {
coefficients: [-4948.31942, 26870.91986, -58334.48399, 63280.17197, -34298.01053, 7430.30146],
reachable: [0.811, 1],
},
},
textOnPanel: {
dark: { coefficients: [0.86713, 0.05232, -0.89428, 4.79014, -5.5432, 1.75023], reachable: [0, 0.542] },
light: {
coefficients: [-8570.89457, 43954.60805, -90084.00702, 92220.6791, -47152.15802, 9632.27113],
reachable: [0.867, 1],
},
},
text: {
dark: { coefficients: [0.89242, 0.02311, -0.44862, 2.34417, -0.06084, -2.63844], reachable: [0, 0.5] },
light: {
coefficients: [-2004.67048, 6664.47299, -6060.70202, -1792.61209, 5133.82359, -1939.85583],
reachable: [0.894, 1],
},
},
} satisfies Record<string, Record<ThemeAppearance, Curve>>;
type Level = keyof typeof LEVELS;
type Surface = ThemeBg | "background" | "scrollbarTrack";
interface Rule {
token: ThemeToken;
on: Surface[];
level: Level;
}
const TOOL_PANELS: Surface[] = ["toolPendingBg", "toolSuccessBg", "toolErrorBg"];
const MESSAGE_PANELS: Surface[] = ["userMessageBg", "customMessageBg"];
const PANELS: ThemeBg[] = [
"userMessageBg",
"toolPendingBg",
"toolSuccessBg",
"toolErrorBg",
"selectedBg",
"searchMatchBg",
"customMessageBg",
];
const THINKING: ThemeColor[] = [
"thinkingOff",
"thinkingMinimal",
"thinkingLow",
"thinkingMedium",
"thinkingHigh",
"thinkingXhigh",
"thinkingMax",
];
const THINKING_LEVELS: Level[] = [
"thinking0",
"thinking1",
"thinking2",
"thinking3",
"thinking4",
"thinking5",
"thinking6",
];
const each = (tokens: ThemeToken[], on: Surface[], level: Level): Rule[] =>
tokens.map((token) => ({ token, on, level }));
const RULES: Rule[] = [
...each(PANELS, ["background"], "panel"),
{ token: "text", on: ["background"], level: "text" },
{ token: "text", on: ["selectedBg"], level: "textOnPanel" },
{ token: "userMessageText", on: ["userMessageBg"], level: "textOnPanel" },
{ token: "toolTitle", on: TOOL_PANELS, level: "textOnPanel" },
...each(["accent", "success", "error", "warning"], ["background", "selectedBg", ...TOOL_PANELS], "readable"),
{ token: "muted", on: ["background", "selectedBg", "customMessageBg", ...TOOL_PANELS], level: "readable" },
{ token: "dim", on: ["background", "selectedBg", "customMessageBg", ...TOOL_PANELS], level: "subtle" },
{ token: "thinkingText", on: ["background"], level: "readable" },
{ token: "customMessageText", on: ["customMessageBg", ...TOOL_PANELS], level: "readable" },
{
token: "customMessageLabel",
on: ["background", "customMessageBg", "selectedBg", ...TOOL_PANELS],
level: "readable",
},
{ token: "toolOutput", on: ["background", ...TOOL_PANELS], level: "readable" },
...each(
["mdHeading", "mdLink", "mdLinkUrl", "mdCode", "mdQuote", "mdCodeBlockBorder", "mdListBullet"],
["background", ...MESSAGE_PANELS],
"readable",
),
{ token: "mdCodeBlock", on: ["background", ...MESSAGE_PANELS, ...TOOL_PANELS], level: "readable" },
...each(["toolDiffAdded", "toolDiffRemoved", "toolDiffContext"], ["background", ...TOOL_PANELS], "readable"),
...each(
[
"syntaxComment",
"syntaxKeyword",
"syntaxFunction",
"syntaxVariable",
"syntaxString",
"syntaxNumber",
"syntaxType",
"syntaxOperator",
"syntaxPunctuation",
],
["background", ...MESSAGE_PANELS, ...TOOL_PANELS],
"readable",
),
{ token: "searchMatchText", on: ["searchMatchBg"], level: "readable" },
...each(["bashMode", "border", "borderAccent"], ["background"], "readable"),
{ token: "borderMuted", on: ["background"], level: "subtle" },
...each(["mdQuoteBorder", "mdHr"], ["background", ...MESSAGE_PANELS, ...TOOL_PANELS], "readable"),
{ token: "scrollbarTrack", on: ["background"], level: "track" },
{ token: "scrollbarThumb", on: ["scrollbarTrack"], level: "thumb" },
...THINKING.map((token, index): Rule => ({ token, on: ["background"], level: THINKING_LEVELS[index] })),
];
/** Relaxation compresses levels stronger than this one toward it before weakening all levels. */
const READABLE_FLOOR: Record<ThemeAppearance, Level> = { dark: "readable", light: "subtle" };
/** Body text uses the terminal's foreground when it reaches this level, which is clearly stronger than muted. */
const FOREGROUND_LEVEL: Level = "emphasis";
/** Text-level tokens that take the terminal's foreground. */
const FOREGROUND_TOKENS: ThemeColor[] = ["text", "userMessageText", "toolTitle"];
/** WCAG 2 contrast ratio that body text must reach on the surfaces it is drawn on. */
const TEXT_MINIMUM_WCAG_CONTRAST = 4.5;
/** Tokens in dependency order: every surface before the tokens drawn on it. */
const SOLVE_ORDER: ThemeToken[] = (() => {
const order: ThemeToken[] = [];
const visit = (token: ThemeToken): void => {
if (order.includes(token)) return;
for (const rule of RULES) {
if (rule.token !== token) continue;
for (const surface of rule.on) if (surface !== "background") visit(surface);
}
order.push(token);
};
for (const rule of RULES) visit(rule.token);
return order;
})();
// ============================================================================
// Public API
// ============================================================================
export interface SystemThemeInput {
foreground?: RgbColor;
background?: RgbColor;
/** ANSI colors 0-15. */
palette?: RgbColor[];
/** Saturation multiplier from 0 (grayscale) to 1. The first frame renders in grayscale until colors arrive. */
saturation?: number;
/** Appearance when the terminal did not report its background, e.g. from its light/dark report or COLORFGBG. */
appearanceHint?: ThemeAppearance;
}
export interface SystemThemeColors {
/** Hex colors, ANSI palette indices, or "" for the terminal default. */
colors: Record<ThemeToken, string | number>;
/** Foreground tokens rendered faint (SGR 2), for terminals that did not report colors. */
dim: ThemeColor[];
appearance: ThemeAppearance | undefined;
}
/** OKLab lightness of an sRGB color, 0-1. */
function oklabLightness(color: RgbColor): number {
return colorToOklch(rgbColor(color.r, color.g, color.b)).l;
}
/** WCAG 2 relative luminance. */
export function relativeLuminance({ r, g, b }: RgbColor): number {
const linear = (channel: number) => {
const value = channel / 255;
return value <= 0.04045 ? value / 12.92 : ((value + 0.055) / 1.055) ** 2.4;
};
return 0.2126 * linear(r) + 0.7152 * linear(g) + 0.0722 * linear(b);
}
/** WCAG 2 contrast ratio, 1-21. */
export function wcagContrast(first: RgbColor, second: RgbColor): number {
const a = relativeLuminance(first);
const b = relativeLuminance(second);
return (Math.max(a, b) + 0.05) / (Math.min(a, b) + 0.05);
}
/**
* Whether a terminal is dark or light, from its reported colors: the direction of its own foreground
* when text can be readable that way, otherwise dark when white text has more contrast on the
* background than black text.
*/
export function terminalAppearance(background: RgbColor, foreground?: RgbColor): ThemeAppearance {
const white = { r: 255, g: 255, b: 255 };
const black = { r: 0, g: 0, b: 0 };
const whiteContrast = wcagContrast(white, background);
const blackContrast = wcagContrast(black, background);
if (foreground) {
const foregroundL = oklabLightness(foreground);
const backgroundL = oklabLightness(background);
if (Math.abs(foregroundL - backgroundL) > 0.05) {
const appearance = foregroundL > backgroundL ? "dark" : "light";
const best = appearance === "dark" ? whiteContrast : blackContrast;
if (best >= TEXT_MINIMUM_WCAG_CONTRAST) return appearance;
}
}
return whiteContrast >= blackContrast ? "dark" : "light";
}
// ============================================================================
// Generation
// ============================================================================
const clamp = (value: number, min: number, max: number) => Math.min(max, Math.max(min, value));
function hexOf({ r, g, b }: RgbColor): string {
return `#${[r, g, b].map((channel) => Math.round(channel).toString(16).padStart(2, "0")).join("")}`;
}
/** Saturation weight at a lightness: a Gaussian (center 0.5, sigma 0.25), 0 at black and white, 1 in the middle. */
function bellWeight(lightness: number): number {
const gaussian = (x: number): number => Math.exp(-((x - 0.5) ** 2) / (2 * 0.25 ** 2));
return (gaussian(lightness) - gaussian(0)) / (1 - gaussian(0));
}
/** A family's saturation curve relative to its maximum: 1 at mid lightness, `min / max` at black and white. */
function saturationCurve({ saturation: { min, max } }: Family, lightness: number): number {
const floor = max > 0 ? min / max : 1;
return floor + (1 - floor) * bellWeight(lightness);
}
/** The target lightness for a level on a surface, or undefined where the level cannot be reached. */
function levelTarget(level: Level, appearance: ThemeAppearance, surfaceL: number): number | undefined {
const curve: Curve = LEVELS[level][appearance];
if (surfaceL < curve.reachable[0] || surfaceL > curve.reachable[1]) return undefined;
return curve.coefficients.reduce((sum, coefficient, power) => sum + coefficient * surfaceL ** power, 0);
}
/**
* Generate the system theme's colors from the terminal's reported colors.
*/
export function generateSystemThemeColors(input: SystemThemeInput): SystemThemeColors {
const saturation = clamp(input.saturation ?? 1, 0, 1);
const { background, foreground } = input;
if (!background) return indexedColors(saturation, input.appearanceHint);
const palette = input.palette?.length === 16 ? input.palette.map(okhslOf) : undefined;
const appearance = terminalAppearance(background, foreground);
const lighter = appearance === "dark";
const extreme = lighter ? 1 : 0;
const backgroundL = oklabLightness(background);
/**
* A token's color at an OKLab lightness. With a palette, the palette color's saturation applies at its
* own lightness and falls off toward black and white along the family's curve, never rising above it.
*/
const paint = (token: ThemeToken, oklabL: number): RgbColor => {
const lightness = oklabToOkhslLightness(oklabL);
const family: Family = FAMILIES[TOKEN_FAMILIES[token]];
if (!palette) {
const { min, max } = family.saturation;
return okhslColor(family.hue, (min + (max - min) * bellWeight(lightness)) * saturation, lightness);
}
return anchored(palette[TOKEN_SLOTS[token] ?? family.slot], family, lightness, saturation);
};
/**
* The lightness a rule needs on a surface, relaxed by `t`: from 0 to 1, levels stronger than the readable
* floor move toward it; from 1 to 2, all levels move toward the surface itself.
*/
const target = (level: Level, surfaceL: number, t: number): number | undefined => {
const reached = levelTarget(level, appearance, surfaceL);
if (reached === undefined && t === 0) return undefined;
const distance = (reached ?? extreme) - surfaceL;
const floor = (levelTarget(READABLE_FLOOR[appearance], appearance, surfaceL) ?? extreme) - surfaceL;
const compressed =
Math.abs(distance) > Math.abs(floor) ? distance - (distance - floor) * Math.min(t, 1) : distance;
return surfaceL + compressed * (1 - Math.max(0, t - 1));
};
/**
* Keep a panel light enough (or dark enough) that white (or black) text still reaches the body text
* minimum on it. This only matters for backgrounds near mid-gray, where it barely does on the background.
*/
const extremeText = lighter ? { r: 255, g: 255, b: 255 } : { r: 0, g: 0, b: 0 };
const readable = (color: RgbColor) => wcagContrast(extremeText, color) >= TEXT_MINIMUM_WCAG_CONTRAST;
const limitPanel = (token: ThemeToken, l: number): RgbColor => {
const color = paint(token, l);
if (readable(color)) return color;
let [low, high] = [backgroundL, l];
for (let index = 0; index < 20; index++) {
const middle = (low + high) / 2;
if (readable(paint(token, middle))) low = middle;
else high = middle;
}
return paint(token, low);
};
const solve = (t: number): Map<Surface | ThemeToken, RgbColor> | undefined => {
const colors = new Map<Surface | ThemeToken, RgbColor>([["background", background]]);
for (const token of SOLVE_ORDER) {
const targets: number[] = [];
for (const rule of RULES) {
if (rule.token !== token) continue;
for (const surface of rule.on) {
const value = target(rule.level, oklabLightness(colors.get(surface) ?? background), t);
if (value === undefined || value < 0 || value > 1) return undefined;
targets.push(value);
}
}
const l = lighter ? Math.max(...targets) : Math.min(...targets);
colors.set(token, PANELS.includes(token as ThemeBg) ? limitPanel(token, l) : paint(token, l));
}
return colors;
};
let relaxation = 0;
let colors = solve(0);
if (!colors) {
// Mid-gray backgrounds cannot fit every level: relax as little as possible. Full relaxation always fits.
let [low, high] = [0, 2];
colors = solve(high);
for (let index = 0; index < 20; index++) {
const middle = (low + high) / 2;
const attempt = solve(middle);
if (attempt) [high, colors] = [middle, attempt];
else low = middle;
}
relaxation = high;
}
const solved = colors ?? new Map<Surface | ThemeToken, RgbColor>();
const surfacesOf = (token: ThemeToken) =>
RULES.filter((rule) => rule.token === token).flatMap((rule) =>
rule.on.map((surface) => solved.get(surface) ?? background),
);
const result = {} as Record<ThemeToken, string | number>;
for (const token of Object.keys(TOKEN_FAMILIES) as ThemeToken[]) {
const color = solved.get(token);
result[token] = color ? hexOf(color) : "";
}
for (const token of FOREGROUND_TOKENS) {
const surfaces = surfacesOf(token);
// Body text uses the terminal's own foreground where it is clearly stronger than muted text; otherwise
// the foreground's hue at just enough lightness.
let text = solved.get(token);
if (foreground) {
const targets = surfaces.map((surface) => target(FOREGROUND_LEVEL, oklabLightness(surface), relaxation));
if (targets.every((value) => value !== undefined && value >= 0 && value <= 1)) {
const needed = lighter ? Math.max(...(targets as number[])) : Math.min(...(targets as number[]));
const foregroundL = oklabLightness(foreground);
if (lighter ? foregroundL >= needed : foregroundL <= needed) {
result[token] = "";
continue;
}
text = anchored(okhslOf(foreground), FAMILIES.neutral, oklabToOkhslLightness(needed), saturation);
}
}
// Body text keeps at least 4.5:1 on the surfaces it is drawn on, even on relaxed mid-gray backgrounds.
if (text) result[token] = hexOf(withTextContrast(text, surfaces, lighter));
}
return { colors: result, dim: [], appearance };
}
function okhslOf({ r, g, b }: RgbColor): OkhslChannels {
return colorToOkhsl(rgbColor(r, g, b));
}
/**
* A source color's hue at another OKHSL lightness. Its saturation applies at its own lightness and falls off
* toward black and white along the family's saturation curve, never rising above it.
*/
function anchored(source: OkhslChannels, family: Family, lightness: number, saturation: number): RgbColor {
const anchor = saturationCurve(family, source.l);
const falloff = anchor > 0 ? Math.min(1, saturationCurve(family, lightness) / anchor) : 1;
return okhslColor(source.h, source.s * falloff * saturation, lightness);
}
/** Move a text color toward white or black until it reaches the WCAG minimum on every surface. */
function withTextContrast(color: RgbColor, surfaces: RgbColor[], lighter: boolean): RgbColor {
const meets = (candidate: RgbColor) =>
surfaces.every((surface) => wcagContrast(candidate, surface) >= TEXT_MINIMUM_WCAG_CONTRAST);
if (meets(color)) return color;
const { h, s, l } = okhslOf(color);
const at = (lightness: number) => okhslColor(h, s, lightness);
const extreme = lighter ? 1 : 0;
if (!meets(at(extreme))) return at(extreme);
let [low, high] = [l, extreme];
for (let index = 0; index < 20; index++) {
const middle = (low + high) / 2;
if (meets(at(middle))) high = middle;
else low = middle;
}
return at(high);
}
/**
* Colors for terminals that reported nothing: the terminal renders ANSI indices 0-15 and the default
* colors with its own theme, so they fit any background. Neutral tokens below body text are faint (SGR 2)
* instead of bright black, which some themes make nearly invisible. Panels have no background.
*/
function indexedColors(saturation: number, appearance: ThemeAppearance | undefined): SystemThemeColors {
const colors = {} as Record<ThemeToken, string | number>;
const dim: ThemeColor[] = [];
for (const [token, familyName] of Object.entries(TOKEN_FAMILIES) as [ThemeToken, FamilyName][]) {
if (PANELS.includes(token as ThemeBg)) {
colors[token] = "";
continue;
}
const neutral = familyName === "neutral";
colors[token] = !neutral && saturation > 0 ? (TOKEN_SLOTS[token] ?? FAMILIES[familyName].slot) : "";
if (neutral && !FOREGROUND_TOKENS.includes(token as ThemeColor)) dim.push(token as ThemeColor);
}
return { colors, dim, appearance };
}

View file

@ -1,13 +1,14 @@
import type { RgbColor, TUI } from "@earendil-works/pi-tui";
import type { RgbColor, TerminalColors, TUI } from "@earendil-works/pi-tui";
import type { SettingsManager } from "../../../core/settings-manager.ts";
import {
detectTerminalBackgroundFromEnv,
detectTerminalBackgroundTheme,
detectTerminalThemeForAuto,
getTerminalTheme,
initTheme,
markTerminalColorsPending,
parseAutoThemeSetting,
resolveThemeSetting,
setTerminalDefaultColors,
SYSTEM_THEME_NAME,
setTerminalColorScheme,
setTerminalColors,
setTheme,
setThemeInstance,
type TerminalTheme,
@ -16,24 +17,57 @@ import {
type ThemeResult = { success: boolean; error?: string };
/**
* How long the system theme stays grayscale before falling back to palette indices. Terminals answer
* the trailing DA1 request right after the color replies, so this only matters for terminals that
* answer neither. Replies arriving later still apply.
*/
const TERMINAL_QUERY_TIMEOUT_MS = 100;
/**
* Query the terminal's colors and pass them to `apply` when the query completes or times out, and again
* if the terminal answers after the timeout. A failed query applies no colors. Settles after the first apply.
*/
export function requestTerminalColors(ui: TUI, apply: (colors: TerminalColors) => void): Promise<void> {
let query: Promise<TerminalColors>;
try {
query = ui.queryTerminalColors({ timeoutMs: TERMINAL_QUERY_TIMEOUT_MS, onLateReply: apply });
} catch {
// Treat a failed query like a terminal that does not report colors.
query = Promise.resolve({});
}
return query.then(apply, () => apply({}));
}
function sameRgb(a: RgbColor | undefined, b: RgbColor | undefined): boolean {
return a === b || (a !== undefined && b !== undefined && a.r === b.r && a.g === b.g && a.b === b.b);
}
function sameTerminalColors(a: TerminalColors, b: TerminalColors): boolean {
if (!sameRgb(a.foreground, b.foreground) || !sameRgb(a.background, b.background)) return false;
if (a.palette === b.palette) return true;
if (!a.palette || !b.palette || a.palette.length !== b.palette.length) return false;
return a.palette.every((color, index) => sameRgb(color, b.palette?.[index]));
}
/**
* Applies the theme setting and keeps it in sync with the terminal. The theme applies immediately, and the
* terminal's colors update it when they arrive; the system theme renders in grayscale until then. Callers
* that bake theme colors into content can wait for the colors with `waitForTerminalColors()`.
*/
export class InteractiveThemeController {
private readonly ui: TUI;
private readonly getSettingsManager: () => SettingsManager;
private readonly showError: (message: string) => void;
private readonly onChanged: () => void;
private currentThemeSetting: string | undefined;
private terminalTheme: TerminalTheme = detectTerminalBackgroundFromEnv().theme;
// Last reported default colors; a query that times out keeps them instead of erasing them.
private terminalColors: { foreground?: RgbColor; background?: RgbColor } = {};
// Last reported colors; a query that times out keeps them instead of erasing them.
private terminalColors: TerminalColors | undefined;
private activeThemeName: string | undefined;
private autoSyncEnabled = false;
private terminalColorSchemeUnsubscribe: (() => void) | undefined;
// Settles when the latest color query completed or timed out, and its colors applied.
private terminalColorQuery: Promise<void> = Promise.resolve();
constructor(
ui: TUI,
@ -49,10 +83,9 @@ export class InteractiveThemeController {
this.showError = options.showError;
this.onChanged = options.onChanged;
this.currentThemeSetting = options.initialThemeSetting;
this.activeThemeName = resolveThemeSetting(
this.currentThemeSetting ?? this.getSettingsManager().getThemeSetting(),
this.terminalTheme,
);
this.activeThemeName = this.resolveThemeName();
// The system theme starts in grayscale; color follows once the terminal reports its colors.
markTerminalColorsPending();
initTheme(this.activeThemeName, true);
this.bindTerminalColorSchemeListener();
}
@ -63,36 +96,25 @@ export class InteractiveThemeController {
this.ui.setTerminalColorSchemeNotifications(this.autoSyncEnabled);
}
async applyFromSettings(): Promise<void> {
const settingsManager = this.getSettingsManager();
const themeSetting = this.currentThemeSetting ?? settingsManager.getThemeSetting();
const autoTheme = parseAutoThemeSetting(themeSetting);
// Theme detection reuses the background reply of the default color query.
const background = this.queryTerminalDefaultColors();
const detector = {
queryTerminalBackgroundColor: () => background,
queryTerminalColorScheme: (options: { timeoutMs: number }) => this.ui.queryTerminalColorScheme(options),
};
if (autoTheme) {
this.terminalTheme = await detectTerminalThemeForAuto({ ui: detector, timeoutMs: TERMINAL_QUERY_TIMEOUT_MS });
this.setAutoSync(true);
this.applyThemeName(this.terminalTheme === "light" ? autoTheme.lightTheme : autoTheme.darkTheme, true);
return;
}
/**
* Apply the theme setting now and query the terminal's colors, which update the theme when they arrive.
* Theme pairs and the system theme follow terminal appearance changes.
*/
applyFromSettings(): void {
const themeSetting = this.getThemeSetting();
const themeName = this.resolveThemeName();
this.setAutoSync(parseAutoThemeSetting(themeSetting) !== undefined || themeName === SYSTEM_THEME_NAME);
this.applyThemeName(themeName, themeSetting !== undefined);
this.queryTerminalColors();
}
this.setAutoSync(false);
if (themeSetting !== undefined) {
this.applyThemeName(themeSetting, true);
return;
}
const detection = await detectTerminalBackgroundTheme({ ui: detector, timeoutMs: TERMINAL_QUERY_TIMEOUT_MS });
this.terminalTheme = detection.theme;
if (!this.applyThemeName(detection.theme).success) return;
if (detection.confidence === "high") {
settingsManager.setTheme(detection.theme);
await settingsManager.flush();
}
/**
* Wait until the latest color query completed or timed out. Content that bakes theme colors into
* strings, such as the startup header, should be built after this. Terminals answer the DA1 request
* right after the color replies, so this only takes the full timeout when a terminal answers nothing.
*/
waitForTerminalColors(): Promise<void> {
return this.terminalColorQuery;
}
getThemeSelection(): string | undefined {
@ -100,7 +122,7 @@ export class InteractiveThemeController {
}
setThemeName(themeName: string, showError = false): ThemeResult {
this.setAutoSync(false);
this.setAutoSync(themeName === SYSTEM_THEME_NAME);
const result = this.applyThemeName(themeName, showError);
if (result.success) {
this.currentThemeSetting = themeName;
@ -108,9 +130,9 @@ export class InteractiveThemeController {
return result;
}
async setThemeSetting(themeSetting: string): Promise<void> {
setThemeSetting(themeSetting: string): void {
this.currentThemeSetting = themeSetting;
await this.applyFromSettings();
this.applyFromSettings();
}
setThemeInstance(themeInstance: Theme): ThemeResult {
@ -122,7 +144,7 @@ export class InteractiveThemeController {
}
preview(themeSettingOrName: string): void {
const themeName = resolveThemeSetting(themeSettingOrName, this.terminalTheme) ?? this.activeThemeName;
const themeName = resolveThemeSetting(themeSettingOrName, getTerminalTheme()) ?? this.activeThemeName;
if (!themeName) return;
if (setTheme(themeName, true).success) {
this.ui.invalidate();
@ -141,46 +163,63 @@ export class InteractiveThemeController {
}
getTerminalTheme(): TerminalTheme {
return this.terminalTheme;
return getTerminalTheme();
}
private getThemeSetting(): string | undefined {
return this.currentThemeSetting ?? this.getSettingsManager().getThemeSetting();
}
/** The theme for the current setting and terminal appearance. Without a setting, pi uses the system theme. */
private resolveThemeName(): string {
return resolveThemeSetting(this.getThemeSetting(), getTerminalTheme()) ?? SYSTEM_THEME_NAME;
}
private applyThemeName(themeName: string, showError = false): ThemeResult {
const result = setTheme(themeName, true);
this.activeThemeName = result.success ? themeName : "dark";
this.activeThemeName = result.success ? themeName : SYSTEM_THEME_NAME;
this.notifyChanged();
if (!result.success && showError) {
this.showError(`Failed to load theme "${themeName}": ${result.error}\nFell back to dark theme.`);
this.showError(`Failed to load theme "${themeName}": ${result.error}\nFell back to the system theme.`);
}
return result;
}
/**
* Query the terminal's default colors, which themes use for tokens set to "". Startup does not
* wait for the replies; the UI re-renders when they arrive. Returns the background reply.
*/
private queryTerminalDefaultColors(): Promise<RgbColor | undefined> {
const timeoutMs = TERMINAL_QUERY_TIMEOUT_MS;
const foreground = this.ui.queryTerminalForegroundColor({ timeoutMs });
const background = this.ui.queryTerminalBackgroundColor({ timeoutMs });
void Promise.all([foreground, background]).then(([foregroundColor, backgroundColor]) => {
const previous = this.terminalColors;
const next = {
foreground: foregroundColor ?? previous.foreground,
background: backgroundColor ?? previous.background,
};
// Re-rendering rebuilds every component, so skip it when nothing changed (including timeouts).
if (sameRgb(next.foreground, previous.foreground) && sameRgb(next.background, previous.background)) return;
this.terminalColors = next;
setTerminalDefaultColors(next);
this.ui.invalidate();
this.ui.requestRender();
});
return background;
/** Query the terminal's colors without waiting for them; `waitForTerminalColors()` waits for this query. */
private queryTerminalColors(): void {
this.terminalColorQuery = requestTerminalColors(this.ui, (colors) => this.applyTerminalColors(colors));
}
private notifyChanged(): void {
/**
* Record reported colors: themes use the default colors for tokens set to "", the system theme is
* generated from all of them, and light/dark detection uses them. Re-renders only when they changed.
*/
private applyTerminalColors(reported: TerminalColors): void {
const previous = this.terminalColors;
const next: TerminalColors = {
foreground: reported.foreground ?? previous?.foreground,
background: reported.background ?? previous?.background,
palette: reported.palette ?? previous?.palette,
};
// Re-rendering rebuilds every component, so skip it when nothing changed (including timeouts).
if (previous && sameTerminalColors(previous, next)) return;
this.terminalColors = next;
setTerminalColors(next);
this.reapplyForTerminal();
this.ui.invalidate();
this.onChanged();
this.ui.requestRender();
}
/**
* Re-apply the setting after the terminal's colors or appearance changed: regenerate the system theme,
* or switch the theme of a pair. Themes set through extensions or previews are left alone.
*/
private reapplyForTerminal(): void {
if (this.activeThemeName === "<in-memory>") return;
const themeName = this.resolveThemeName();
if (themeName === SYSTEM_THEME_NAME || themeName !== this.activeThemeName) {
this.applyThemeName(themeName);
}
}
private setAutoSync(enabled: boolean): void {
@ -191,23 +230,24 @@ export class InteractiveThemeController {
private bindTerminalColorSchemeListener(): void {
this.terminalColorSchemeUnsubscribe = this.ui.onTerminalColorSchemeChange((terminalTheme) =>
this.applyTerminalTheme(terminalTheme),
this.applyTerminalColorSchemeChange(terminalTheme),
);
}
private applyTerminalTheme(terminalTheme: TerminalTheme): void {
/**
* The terminal reported a light/dark switch. Its colors changed too, so query them again: they decide
* the appearance. The reported scheme only matters for terminals that do not report their background.
*/
private applyTerminalColorSchemeChange(terminalTheme: TerminalTheme): void {
if (!this.autoSyncEnabled) return;
this.terminalTheme = terminalTheme;
// Switching light/dark changes the terminal's default colors too.
void this.queryTerminalDefaultColors();
const autoTheme = parseAutoThemeSetting(this.currentThemeSetting ?? this.getSettingsManager().getThemeSetting());
if (!autoTheme) {
this.setAutoSync(false);
return;
}
const themeName = terminalTheme === "light" ? autoTheme.lightTheme : autoTheme.darkTheme;
if (themeName !== this.activeThemeName) {
this.applyThemeName(themeName);
}
const previous = getTerminalTheme();
setTerminalColorScheme(terminalTheme);
if (getTerminalTheme() !== previous) this.reapplyForTerminal();
this.queryTerminalColors();
}
private notifyChanged(): void {
this.ui.invalidate();
this.onChanged();
}
}

View file

@ -11,7 +11,7 @@ import { type Static, Type } from "typebox";
import { Compile } from "typebox/compile";
const ColorValueSchema = Type.Union([
Type.String(), // hex, OKLCH, var ref "primary", or empty ""
Type.String(), // hex, OKLCH, OKHSL, var ref "primary", or empty ""
Type.Integer({ minimum: 0, maximum: 255 }), // 256-color index
]);

View file

@ -26,7 +26,7 @@
"oneOf": [
{
"type": "string",
"description": "Hex color (#RGB or #RRGGBB), OKLCH color, variable reference, or empty string for terminal default"
"description": "Hex color (#RGB or #RRGGBB), OKLCH or OKHSL color, variable reference, or empty string for terminal default"
},
{
"type": "integer",
@ -347,7 +347,7 @@
"oneOf": [
{
"type": "string",
"description": "Hex color (#RGB or #RRGGBB), OKLCH color, variable reference, or empty string for terminal default"
"description": "Hex color (#RGB or #RRGGBB), OKLCH or OKHSL color, variable reference, or empty string for terminal default"
},
{
"type": "integer",

View file

@ -6,12 +6,12 @@ import {
type Color,
colorToHex,
colorToOklch,
colorToRgb,
type EditorTheme,
foregroundAnsi,
getTerminalColorMode,
indexedColor,
type MarkdownTheme,
mixColors,
parseColor,
type RgbColor,
rgbColor,
@ -19,6 +19,7 @@ import {
type SettingsListTheme,
styleTextWithAnsi,
type TerminalColorMode,
type TerminalColors,
type TextAttributes,
} from "@earendil-works/pi-tui";
import chalk from "chalk";
@ -27,6 +28,9 @@ import type { SourceInfo } from "../../../core/source-info.ts";
import { closeWatcher, watchWithErrorHandler } from "../../../utils/fs-watch.ts";
import { highlight, supportsLanguage } from "../../../utils/syntax-highlight.ts";
import { stripBom } from "../../../utils/text.ts";
import { generateSystemThemeColors, SYSTEM_THEME_NAME, terminalAppearance } from "./system-theme.ts";
export { SYSTEM_THEME_NAME } from "./system-theme.ts";
// ============================================================================
// Types & Schema
@ -133,7 +137,7 @@ function resolveVarRefs(
vars: Record<string, ColorValue>,
visited = new Set<string>(),
): string | number {
if (typeof value === "number" || value === "" || value.startsWith("#") || /^oklch\(/i.test(value)) {
if (typeof value === "number" || value === "" || value.startsWith("#") || /^ok(lch|hsl)\(/i.test(value)) {
return value;
}
if (visited.has(value)) {
@ -181,22 +185,35 @@ function withThemeColorFallbacks(colors: ThemeJson["colors"]): ThemeJson["colors
/** The background a theme is designed for. */
export type ThemeAppearance = TerminalTheme;
interface TerminalDefaultColors {
foreground?: Color;
background?: Color;
// The terminal's reported colors. Replaced (never mutated) on update, so themes can cache resolved colors
// by identity.
let terminalColors: TerminalColors = {};
// While the terminal color query is in flight, the system theme renders in grayscale.
let terminalColorsPending = false;
// The terminal's last light/dark report (mode 2031). Only used while it has not reported a background.
let terminalColorScheme: TerminalTheme | undefined;
/**
* Record the terminal's reported colors. Themes use the default colors for tokens set to "" (terminal
* default); the system theme is generated from all of them. Ends the pending state.
*/
export function setTerminalColors(colors: TerminalColors): void {
terminalColors = { ...colors };
terminalColorsPending = false;
}
// Replaced (never mutated) on update, so themes can cache resolved colors by identity.
let terminalDefaultColors: TerminalDefaultColors = {};
/** Record the terminal's light/dark report, the fallback for terminals that do not report their background. */
export function setTerminalColorScheme(scheme: TerminalTheme | undefined): void {
terminalColorScheme = scheme;
}
/** Record the terminal's reported default colors. Themes use them for tokens set to "" (terminal default). */
export function setTerminalDefaultColors(colors: { foreground?: RgbColor; background?: RgbColor }): void {
const toColor = (rgb: RgbColor | undefined) => rgb && rgbColor(rgb.r, rgb.g, rgb.b);
terminalDefaultColors = { foreground: toColor(colors.foreground), background: toColor(colors.background) };
/** Render the system theme in grayscale until `setTerminalColors()` reports the terminal's colors. */
export function markTerminalColorsPending(): void {
terminalColorsPending = true;
}
/** Assumed terminal default colors when the terminal does not report them. */
const GUESSED_DEFAULT_COLORS: Record<ThemeAppearance, Required<TerminalDefaultColors>> = {
const GUESSED_DEFAULT_COLORS: Record<ThemeAppearance, { foreground: Color; background: Color }> = {
dark: { foreground: parseColor("#e5e5e7"), background: parseColor("#000000") },
light: { foreground: parseColor("#000000"), background: parseColor("#ffffff") },
};
@ -234,8 +251,10 @@ export class Theme {
private readonly concreteColors: Partial<Record<ThemeToken, Color>> = {};
private readonly defaultForegroundTokens: ThemeToken[] = [];
private readonly defaultBackgroundTokens: ThemeToken[] = [];
// Foreground tokens rendered faint (SGR 2) on top of their color.
private readonly dimTokens: ReadonlySet<ThemeColor>;
private readonly ownAppearance: ThemeAppearance | undefined;
private resolvedColors: { terminal: TerminalDefaultColors; colors: Readonly<Record<ThemeToken, Color>> } | undefined;
private resolvedColors: { terminal: TerminalColors; colors: Readonly<Record<ThemeToken, Color>> } | undefined;
constructor(
fgColors: Record<Exclude<ThemeColor, OptionalThemeColor>, string | number> &
@ -243,12 +262,20 @@ export class Theme {
bgColors: Record<Exclude<ThemeBg, OptionalThemeBg>, string | number> &
Partial<Record<OptionalThemeBg, string | number>>,
mode: TerminalColorMode,
options: { name?: string; sourcePath?: string; sourceInfo?: SourceInfo; appearance?: ThemeAppearance } = {},
options: {
name?: string;
sourcePath?: string;
sourceInfo?: SourceInfo;
appearance?: ThemeAppearance;
/** Foreground tokens to render faint (SGR 2). */
dim?: readonly ThemeColor[];
} = {},
) {
this.name = options.name;
this.sourcePath = options.sourcePath;
this.sourceInfo = options.sourceInfo;
this.mode = mode;
this.dimTokens = new Set(options.dim);
const foregrounds = {
...fgColors,
scrollbarTrack: fgColors.scrollbarTrack ?? fgColors.muted,
@ -281,25 +308,32 @@ export class Theme {
/**
* The background the theme is designed for: declared in the theme JSON, detected from its colors,
* or, for themes without usable colors, taken from the terminal background.
* or, for themes without usable colors, the terminal's appearance.
*/
get appearance(): ThemeAppearance {
if (this.ownAppearance) return this.ownAppearance;
const background = terminalDefaultColors.background;
return background ? getThemeForRgbColor(colorToRgb(background)) : "dark";
return this.ownAppearance ?? getTerminalTheme();
}
/**
* Concrete colors for all tokens. Tokens set to "" (terminal default) use the terminal's reported
* default colors, or a guess based on `appearance` when the terminal did not report them.
* default colors, or a guess based on `appearance` when the terminal did not report them. Faint
* tokens are approximated by mixing their color toward the background.
*/
get colors(): Readonly<Record<ThemeToken, Color>> {
const terminal = terminalDefaultColors;
const terminal = terminalColors;
if (this.resolvedColors?.terminal !== terminal) {
const guess = GUESSED_DEFAULT_COLORS[this.appearance];
const toColor = (rgb: RgbColor | undefined, fallback: Color) =>
rgb ? rgbColor(rgb.r, rgb.g, rgb.b) : fallback;
const foreground = toColor(terminal.foreground, guess.foreground);
const background = toColor(terminal.background, guess.background);
const colors = { ...this.concreteColors };
for (const token of this.defaultForegroundTokens) colors[token] = terminal.foreground ?? guess.foreground;
for (const token of this.defaultBackgroundTokens) colors[token] = terminal.background ?? guess.background;
for (const token of this.defaultForegroundTokens) colors[token] = foreground;
for (const token of this.defaultBackgroundTokens) colors[token] = background;
for (const token of this.dimTokens) {
const color = colors[token];
if (color) colors[token] = mixColors(color, background, 0.4);
}
this.resolvedColors = { terminal, colors: Object.freeze(colors as Record<ThemeToken, Color>) };
}
return this.resolvedColors.colors;
@ -307,6 +341,7 @@ export class Theme {
style(text: string, options: ThemeStyle): string {
const { fg, bg } = options;
if (typeof fg === "string" && this.dimTokens.has(fg)) options = { ...options, dim: true };
return styleTextWithAnsi(
text,
fg === undefined
@ -325,6 +360,7 @@ export class Theme {
fg(color: ThemeColor, text: string): string {
const ansi = this.tokenAnsi(this.fgAnsi, color);
if (this.dimTokens.has(color)) return `${ansi}\x1b[2m${text}\x1b[22;39m`;
return `${ansi}${text}\x1b[39m`;
}
@ -359,8 +395,10 @@ export class Theme {
return chalk.strikethrough(text);
}
/** Opening escape sequence for a foreground token. Faint tokens include SGR 2, which `\x1b[22m` closes. */
getFgAnsi(color: ThemeColor): string {
return this.tokenAnsi(this.fgAnsi, color);
const ansi = this.tokenAnsi(this.fgAnsi, color);
return this.dimTokens.has(color) ? `${ansi}\x1b[2m` : ansi;
}
getBgAnsi(color: ThemeBg): string {
@ -438,7 +476,8 @@ export function getAvailableThemesWithPaths(): ThemeInfo[] {
result.push(themeInfo);
};
// Built-in themes
// Built-in themes. The system theme is generated, so it has no file.
addTheme({ name: SYSTEM_THEME_NAME, path: undefined });
for (const name of Object.keys(getBuiltinThemes())) {
addTheme({ name, path: path.join(themesDir, `${name}.json`) });
}
@ -452,7 +491,10 @@ export function getAvailableThemesWithPaths(): ThemeInfo[] {
addTheme({ name, path: theme.sourcePath });
}
return result.sort((a, b) => a.name.localeCompare(b.name));
// The system theme comes first: it is the default and adapts to every terminal.
return result.sort((a, b) =>
a.name === SYSTEM_THEME_NAME ? -1 : b.name === SYSTEM_THEME_NAME ? 1 : a.name.localeCompare(b.name),
);
}
function getCustomThemeInfos(): ThemeInfo[] {
@ -528,27 +570,36 @@ function loadThemeJson(name: string): ThemeJson {
return parseThemeJsonContent(name, content);
}
function createTheme(themeJson: ThemeJson, mode?: TerminalColorMode, sourcePath?: string): Theme {
const colorMode = mode ?? getTerminalColorMode();
const resolvedColors = resolveThemeColors(withThemeColorFallbacks(themeJson.colors), themeJson.vars);
const fgColors: Record<ThemeColor, string | number> = {} as Record<ThemeColor, string | number>;
const bgColors: Record<ThemeBg, string | number> = {} as Record<ThemeBg, string | number>;
const bgColorKeys: Set<string> = new Set([
"selectedBg",
"searchMatchBg",
"userMessageBg",
"customMessageBg",
"toolPendingBg",
"toolSuccessBg",
"toolErrorBg",
]);
for (const [key, value] of Object.entries(resolvedColors)) {
if (bgColorKeys.has(key)) {
const BACKGROUND_TOKENS: ReadonlySet<string> = new Set<ThemeBg>([
"selectedBg",
"searchMatchBg",
"userMessageBg",
"customMessageBg",
"toolPendingBg",
"toolSuccessBg",
"toolErrorBg",
]);
function splitThemeColors(colors: Record<string, string | number>): {
fgColors: Record<ThemeColor, string | number>;
bgColors: Record<ThemeBg, string | number>;
} {
const fgColors = {} as Record<ThemeColor, string | number>;
const bgColors = {} as Record<ThemeBg, string | number>;
for (const [key, value] of Object.entries(colors)) {
if (BACKGROUND_TOKENS.has(key)) {
bgColors[key as ThemeBg] = value;
} else {
fgColors[key as ThemeColor] = value;
}
}
return { fgColors, bgColors };
}
function createTheme(themeJson: ThemeJson, mode?: TerminalColorMode, sourcePath?: string): Theme {
const colorMode = mode ?? getTerminalColorMode();
const resolvedColors = resolveThemeColors(withThemeColorFallbacks(themeJson.colors), themeJson.vars);
const { fgColors, bgColors } = splitThemeColors(resolvedColors);
return new Theme(fgColors, bgColors, colorMode, {
name: themeJson.name,
sourcePath,
@ -556,6 +607,21 @@ function createTheme(themeJson: ThemeJson, mode?: TerminalColorMode, sourcePath?
});
}
/** Generate the system theme from the terminal's reported colors (grayscale while they are pending). */
function createSystemTheme(mode?: TerminalColorMode): Theme {
const generated = generateSystemThemeColors({
...terminalColors,
saturation: terminalColorsPending ? 0 : 1,
appearanceHint: getTerminalTheme(),
});
const { fgColors, bgColors } = splitThemeColors(generated.colors);
return new Theme(fgColors, bgColors, mode ?? getTerminalColorMode(), {
name: SYSTEM_THEME_NAME,
appearance: generated.appearance,
dim: generated.dim,
});
}
export function loadThemeFromPath(themePath: string, mode?: TerminalColorMode): Theme {
const content = fs.readFileSync(themePath, "utf-8");
const themeJson = parseThemeJsonContent(themePath, content);
@ -563,6 +629,8 @@ export function loadThemeFromPath(themePath: string, mode?: TerminalColorMode):
}
function loadTheme(name: string, mode?: TerminalColorMode): Theme {
// The system theme name is reserved: it takes precedence over custom themes of the same name.
if (name === SYSTEM_THEME_NAME) return createSystemTheme(mode);
const registeredTheme = registeredThemes.get(name);
if (registeredTheme) {
return registeredTheme;
@ -611,129 +679,39 @@ export function resolveThemeSetting(
return undefined;
}
export interface TerminalThemeDetection {
theme: TerminalTheme;
source: "terminal background" | "COLORFGBG" | "fallback";
detail: string;
confidence: "high" | "low";
/**
* Dark or light from the `COLORFGBG` environment variable some terminals set, or undefined without a
* usable background index. The value is `fg;bg` or `fg;xpm;bg` (rxvt), where a field is an ANSI color
* index or `default` when the color is not in the palette. The index refers to the terminal's own palette,
* whose colors are unknown here, so it is classified by index like Vim does: 0-6 and 8 (bright black, e.g.
* Solarized Dark's background) are dark, 7 and 9-15 are light.
*/
export function detectColorFgBgTheme(env: NodeJS.ProcessEnv = process.env): TerminalTheme | undefined {
const bg = env.COLORFGBG?.split(";").at(-1)?.trim();
if (!bg || !/^\d{1,2}$/.test(bg)) return undefined;
const index = Number(bg);
if (index > 15) return undefined;
return index <= 6 || index === 8 ? "dark" : "light";
}
export interface TerminalThemeDetectionOptions {
env?: NodeJS.ProcessEnv;
/**
* Whether the terminal is dark or light. The background it renders decides, classified the same way the
* system theme does. Without a reported background: the terminal's light/dark report, then COLORFGBG,
* then dark.
*/
export function detectTerminalTheme(
colors: TerminalColors = {},
reportedScheme?: TerminalTheme,
env: NodeJS.ProcessEnv = process.env,
): TerminalTheme {
const { background, foreground } = colors;
if (background) return terminalAppearance(background, foreground);
return reportedScheme ?? detectColorFgBgTheme(env) ?? "dark";
}
export interface TerminalBackgroundThemeDetector {
queryTerminalBackgroundColor({ timeoutMs }: { timeoutMs: number }): Promise<RgbColor | undefined>;
}
export interface TerminalAutoThemeDetector extends TerminalBackgroundThemeDetector {
queryTerminalColorScheme?({ timeoutMs }: { timeoutMs: number }): Promise<TerminalTheme | undefined>;
}
export interface TerminalBackgroundThemeDetectionOptions extends TerminalThemeDetectionOptions {
ui: TerminalBackgroundThemeDetector;
timeoutMs: number;
}
export interface TerminalAutoThemeDetectionOptions extends TerminalThemeDetectionOptions {
ui: TerminalAutoThemeDetector;
timeoutMs: number;
}
function getColorFgBgBackgroundIndex(colorfgbg: string): number | undefined {
const parts = colorfgbg.split(";");
for (let i = parts.length - 1; i >= 0; i--) {
const bg = parseInt(parts[i].trim(), 10);
if (Number.isInteger(bg) && bg >= 0 && bg <= 255) {
return bg;
}
}
return undefined;
}
function getRgbColorLuminance({ r, g, b }: RgbColor): number {
const toLinear = (channel: number) => {
const value = channel / 255;
return value <= 0.03928 ? value / 12.92 : ((value + 0.055) / 1.055) ** 2.4;
};
return 0.2126 * toLinear(r) + 0.7152 * toLinear(g) + 0.0722 * toLinear(b);
}
function getAnsiColorLuminance(index: number): number {
return getRgbColorLuminance(colorToRgb(indexedColor(index)));
}
export function getThemeForRgbColor(rgb: RgbColor): TerminalTheme {
return getRgbColorLuminance(rgb) >= 0.5 ? "light" : "dark";
}
export function detectTerminalBackgroundFromEnv(options: TerminalThemeDetectionOptions = {}): TerminalThemeDetection {
const env = options.env ?? process.env;
const colorfgbg = env.COLORFGBG || "";
const bg = getColorFgBgBackgroundIndex(colorfgbg);
if (bg !== undefined) {
return {
theme: getAnsiColorLuminance(bg) >= 0.5 ? "light" : "dark",
source: "COLORFGBG",
detail: `background color index ${bg}`,
confidence: "high",
};
}
return {
theme: "dark",
source: "fallback",
detail: "no terminal background hint found",
confidence: "low",
};
}
export async function detectTerminalBackgroundTheme({
ui,
timeoutMs,
env,
}: TerminalBackgroundThemeDetectionOptions): Promise<TerminalThemeDetection> {
try {
const rgb = await ui.queryTerminalBackgroundColor({ timeoutMs });
if (rgb) {
return {
theme: getThemeForRgbColor(rgb),
source: "terminal background",
detail: `OSC 11 background rgb(${rgb.r}, ${rgb.g}, ${rgb.b})`,
confidence: "high",
};
}
} catch {
// Fall back to environment-based detection when the terminal query fails.
}
return detectTerminalBackgroundFromEnv({ env });
}
export async function detectTerminalThemeForAuto({
ui,
timeoutMs,
env,
}: TerminalAutoThemeDetectionOptions): Promise<TerminalTheme> {
let colorSchemePromise: Promise<TerminalTheme | undefined> | undefined;
try {
colorSchemePromise = ui.queryTerminalColorScheme?.({ timeoutMs });
} catch {
// Fall back to OSC 11 / COLORFGBG detection when starting the color-scheme query fails.
}
const backgroundThemePromise = detectTerminalBackgroundTheme({ ui, timeoutMs, env });
try {
const colorScheme = await colorSchemePromise;
if (colorScheme) return colorScheme;
} catch {
// Fall back to the concurrently queried OSC 11 / COLORFGBG detection.
}
return (await backgroundThemePromise).theme;
}
export function getDefaultTheme(): string {
return detectTerminalBackgroundFromEnv().theme;
/** Whether the terminal is dark or light, from everything it reported so far. See `detectTerminalTheme()`. */
export function getTerminalTheme(): TerminalTheme {
return detectTerminalTheme(terminalColors, terminalColorScheme);
}
// ============================================================================
@ -776,7 +754,7 @@ export function setRegisteredThemes(themes: Theme[]): void {
}
export function initTheme(themeName?: string, enableWatcher: boolean = false): void {
const name = themeName ?? getDefaultTheme();
const name = themeName ?? SYSTEM_THEME_NAME;
currentThemeName = name;
try {
setGlobalTheme(loadTheme(name));
@ -784,9 +762,9 @@ export function initTheme(themeName?: string, enableWatcher: boolean = false): v
startThemeWatcher();
}
} catch (_error) {
// Theme is invalid - fall back to dark theme silently
currentThemeName = "dark";
setGlobalTheme(loadTheme("dark"));
// Theme is invalid - fall back to the system theme silently
currentThemeName = SYSTEM_THEME_NAME;
setGlobalTheme(loadTheme(SYSTEM_THEME_NAME));
// Don't start watcher for fallback theme
}
}
@ -803,9 +781,9 @@ export function setTheme(name: string, enableWatcher: boolean = false): { succes
}
return { success: true };
} catch (error) {
// Theme is invalid - fall back to dark theme
currentThemeName = "dark";
setGlobalTheme(loadTheme("dark"));
// Theme is invalid - fall back to the system theme
currentThemeName = SYSTEM_THEME_NAME;
setGlobalTheme(loadTheme(SYSTEM_THEME_NAME));
// Don't start watcher for fallback theme
return {
success: false,
@ -831,7 +809,12 @@ function startThemeWatcher(): void {
stopThemeWatcher();
// Only watch if it's a custom theme (not built-in)
if (!currentThemeName || currentThemeName === "dark" || currentThemeName === "light") {
if (
!currentThemeName ||
currentThemeName === "dark" ||
currentThemeName === "light" ||
currentThemeName === SYSTEM_THEME_NAME
) {
return;
}
@ -918,7 +901,7 @@ export function stopThemeWatcher(): void {
* Used by HTML export to generate CSS custom properties.
*/
export function getResolvedThemeColors(themeName?: string): Record<string, string> {
const colors = loadTheme(themeName ?? currentThemeName ?? getDefaultTheme()).colors;
const colors = loadTheme(themeName ?? currentThemeName ?? SYSTEM_THEME_NAME).colors;
return Object.fromEntries(Object.entries(colors).map(([token, color]) => [token, colorToHex(color)]));
}
@ -926,7 +909,7 @@ export function getResolvedThemeColors(themeName?: string): Record<string, strin
* Check if a theme is a "light" theme (for CSS that needs light/dark variants).
*/
export function isLightTheme(themeName?: string): boolean {
return loadTheme(themeName ?? currentThemeName ?? getDefaultTheme()).appearance === "light";
return loadTheme(themeName ?? currentThemeName ?? SYSTEM_THEME_NAME).appearance === "light";
}
/**
@ -938,19 +921,21 @@ export function getThemeExportColors(themeName?: string): {
cardBg?: string;
infoBg?: string;
} {
const name = themeName ?? currentThemeName ?? getDefaultTheme();
const name = themeName ?? currentThemeName ?? SYSTEM_THEME_NAME;
if (name === SYSTEM_THEME_NAME) return {};
try {
const themeJson = loadThemeJson(name);
const exportSection = themeJson.export;
if (!exportSection) return {};
const vars = themeJson.vars ?? {};
// Export colors end up in CSS, which understands hex, rgb() and oklch() values directly.
// Export colors end up in CSS, which understands hex and oklch() values directly but not okhsl().
const resolve = (value: ColorValue | undefined): string | undefined => {
if (value === undefined) return undefined;
const resolved = resolveVarRefs(value, vars);
if (typeof resolved === "number") return colorToHex(indexedColor(resolved));
if (resolved === "") return undefined;
if (/^okhsl\(/i.test(resolved)) return colorToHex(parseColor(resolved));
return resolved;
};

View file

@ -1,46 +1,34 @@
import { complete, resetApiProviders } from "@earendil-works/pi-ai/compat";
import { describe, expect, it, vi } from "vitest";
import { describe, expect, it } from "vitest";
import { AuthStorage } from "../src/core/auth-storage.ts";
import { ModelRegistry } from "../src/core/model-registry.ts";
import { ModelRuntime } from "../src/core/model-runtime.ts";
const openAIState = vi.hoisted(() => ({ clientOptions: undefined as unknown }));
interface CapturedRequest {
url: string;
headers: Headers;
}
vi.mock("openai", () => {
class FakeOpenAI {
constructor(options: unknown) {
openAIState.clientOptions = options;
}
chat = {
completions: {
create: () => {
const stream = {
async *[Symbol.asyncIterator]() {
yield {
choices: [{ delta: {}, finish_reason: "stop" }],
usage: { prompt_tokens: 1, completion_tokens: 1 },
};
},
};
const promise = Promise.resolve(stream) as Promise<typeof stream> & {
withResponse(): Promise<{
data: typeof stream;
response: { status: number; headers: Headers };
}>;
};
promise.withResponse = async () => ({
data: stream,
response: { status: 200, headers: new Headers() },
});
return promise;
},
},
function createCapturingFetch(): { fetch: typeof globalThis.fetch; requests: CapturedRequest[] } {
const requests: CapturedRequest[] = [];
const fetch: typeof globalThis.fetch = async (input, init) => {
const request = new Request(input, init);
requests.push({ url: request.url, headers: request.headers });
const chunk = {
id: "chatcmpl-test",
object: "chat.completion.chunk",
created: 0,
model: "test",
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
};
}
return { default: FakeOpenAI };
});
return new Response(`data: ${JSON.stringify(chunk)}\n\ndata: [DONE]\n\n`, {
status: 200,
headers: { "content-type": "text/event-stream" },
});
};
return { fetch, requests };
}
async function createCloudflareRuntime(): Promise<{ modelRuntime: ModelRuntime; modelRegistry: ModelRegistry }> {
const authStorage = AuthStorage.inMemory();
@ -56,6 +44,8 @@ async function createCloudflareRuntime(): Promise<{ modelRuntime: ModelRuntime;
return { modelRuntime, modelRegistry: new ModelRegistry(modelRuntime) };
}
const CLOUDFLARE_COMPAT_URL = "https://gateway.ai.cloudflare.com/v1/test-account/test-gateway/compat/chat/completions";
describe("ModelRegistry Cloudflare compat streaming", () => {
it("materializes the Cloudflare endpoint through ModelRuntime streaming", async () => {
const { modelRuntime } = await createCloudflareRuntime();
@ -63,14 +53,13 @@ describe("ModelRegistry Cloudflare compat streaming", () => {
expect(model).toBeDefined();
resetApiProviders();
await modelRuntime.completeSimple(model!, { messages: [] });
const { fetch, requests } = createCapturingFetch();
const result = await modelRuntime.completeSimple(model!, { messages: [] }, { fetch });
const clientOptions = openAIState.clientOptions as {
baseURL?: string;
defaultHeaders?: Record<string, unknown>;
};
expect(clientOptions.baseURL).toBe("https://gateway.ai.cloudflare.com/v1/test-account/test-gateway/compat");
expect(clientOptions.defaultHeaders?.["cf-aig-authorization"]).toBe("Bearer test-token");
expect(result.stopReason).toBe("stop");
expect(requests).toHaveLength(1);
expect(requests[0].url).toBe(CLOUDFLARE_COMPAT_URL);
expect(requests[0].headers.get("cf-aig-authorization")).toBe("Bearer test-token");
});
it("materializes the Cloudflare endpoint after extension-style auth resolution", async () => {
@ -88,15 +77,14 @@ describe("ModelRegistry Cloudflare compat streaming", () => {
"x-api-key": null,
});
await complete(model!, { messages: [] }, auth);
const { fetch, requests } = createCapturingFetch();
const result = await complete(model!, { messages: [] }, { ...auth, fetch });
const clientOptions = openAIState.clientOptions as {
baseURL?: string;
defaultHeaders?: Record<string, unknown>;
};
expect(clientOptions.baseURL).toBe("https://gateway.ai.cloudflare.com/v1/test-account/test-gateway/compat");
expect(clientOptions.defaultHeaders?.["cf-aig-authorization"]).toBe("Bearer test-token");
expect(clientOptions.defaultHeaders?.Authorization).toBeNull();
expect(clientOptions.defaultHeaders?.["x-api-key"]).toBeNull();
expect(result.stopReason).toBe("stop");
expect(requests).toHaveLength(1);
expect(requests[0].url).toBe(CLOUDFLARE_COMPAT_URL);
expect(requests[0].headers.get("cf-aig-authorization")).toBe("Bearer test-token");
expect(requests[0].headers.has("authorization")).toBe(false);
expect(requests[0].headers.has("x-api-key")).toBe(false);
});
});

View file

@ -0,0 +1,103 @@
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { resetCapabilitiesCache, setCapabilityOverrides } from "@earendil-works/pi-tui";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { DefaultResourceLoader } from "../src/core/resource-loader.ts";
import { SettingsManager } from "../src/core/settings-manager.ts";
describe("DefaultResourceLoader theme color mode", () => {
let tempDir: string;
let agentDir: string;
let cwd: string;
let themePath: string;
beforeEach(() => {
tempDir = mkdtempSync(join(tmpdir(), "resource-loader-theme-"));
agentDir = join(tempDir, "agent");
cwd = join(tempDir, "project");
mkdirSync(agentDir, { recursive: true });
mkdirSync(cwd, { recursive: true });
const themeJson = JSON.parse(
readFileSync(join(process.cwd(), "src", "modes", "interactive", "theme", "dark.json"), "utf-8"),
) as { name: string; colors: Record<string, string | number> };
themeJson.name = "capability-test";
themeJson.colors.userMessageBg = "#3c3544";
themePath = join(tempDir, "capability-test.json");
writeFileSync(themePath, JSON.stringify(themeJson));
});
afterEach(() => {
vi.unstubAllEnvs();
setCapabilityOverrides({});
resetCapabilitiesCache();
rmSync(tempDir, { recursive: true, force: true });
});
// Regression test for #9973.
it.each([
{
environment: "256-color",
environmentOverride: "0",
setting: true,
expected: "\x1b[48;2;60;53;68mx\x1b[49m",
},
{
environment: "truecolor",
environmentOverride: "1",
setting: false,
expected: "\x1b[48;5;59mx\x1b[49m",
},
])(
"uses the $setting setting over a $environment environment",
async ({ environmentOverride, setting, expected }) => {
vi.stubEnv("PI_TRUE_COLOR", environmentOverride);
setCapabilityOverrides({});
resetCapabilitiesCache();
const loader = new DefaultResourceLoader({
cwd,
agentDir,
settingsManager: SettingsManager.inMemory({ terminal: { trueColor: setting } }),
additionalThemePaths: [themePath],
noExtensions: true,
noSkills: true,
noPromptTemplates: true,
noContextFiles: true,
});
await loader.reload();
const loadedTheme = loader.getThemes().themes.find((theme) => theme.name === "capability-test");
expect(loadedTheme?.bg("userMessageBg", "x")).toBe(expected);
},
);
it("returns to automatic detection after an explicit setting is removed", async () => {
vi.stubEnv("PI_TRUE_COLOR", "1");
setCapabilityOverrides({});
resetCapabilitiesCache();
const settingsPath = join(agentDir, "settings.json");
writeFileSync(settingsPath, JSON.stringify({ terminal: { trueColor: false } }));
const settingsManager = SettingsManager.create(cwd, agentDir);
const loader = new DefaultResourceLoader({
cwd,
agentDir,
settingsManager,
additionalThemePaths: [themePath],
noExtensions: true,
noSkills: true,
noPromptTemplates: true,
noContextFiles: true,
});
await loader.reload();
setCapabilityOverrides(settingsManager.getTerminalCapabilityOverrides());
writeFileSync(settingsPath, "{}");
await loader.reload();
const loadedTheme = loader.getThemes().themes.find((theme) => theme.name === "capability-test");
expect(loadedTheme?.bg("userMessageBg", "x")).toBe("\x1b[48;2;60;53;68mx\x1b[49m");
});
});

View file

@ -1,9 +1,21 @@
import { constants as bufferConstants } from "buffer";
import { appendFileSync, closeSync, mkdirSync, openSync, readFileSync, rmSync, writeFileSync, writeSync } from "fs";
import {
appendFileSync,
closeSync,
existsSync,
mkdirSync,
mkdtempSync,
openSync,
readFileSync,
rmSync,
writeFileSync,
writeSync,
} from "fs";
import { tmpdir } from "os";
import { join } from "path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { findMostRecentSession, loadEntriesFromFile, SessionManager } from "../../src/core/session-manager.ts";
import { assistantMsg, readSessionFileRoles, userMsg } from "../utilities.ts";
const HEADER_SCAN_LIMIT_BYTES = 1024 * 1024;
@ -420,3 +432,43 @@ describe("SessionManager.setSessionFile with corrupted files", () => {
expect(sm2.getHeader()?.type).toBe("session");
});
});
describe("SessionManager session file creation", () => {
let tempDir: string;
beforeEach(() => {
tempDir = mkdtempSync(join(tmpdir(), "pi-session-persist-"));
});
afterEach(() => {
rmSync(tempDir, { recursive: true, force: true });
});
it("does not create a file for a session with only setup entries", () => {
const session = SessionManager.create(tempDir, tempDir);
session.appendModelChange("anthropic", "claude-sonnet-4-5");
session.appendThinkingLevelChange("off");
expect(existsSync(session.getSessionFile()!)).toBe(false);
});
// #10000: the first prompt must survive a first turn that never produces an assistant message
it("creates the file when the first user message is appended", () => {
const session = SessionManager.create(tempDir, tempDir);
session.appendModelChange("anthropic", "claude-sonnet-4-5");
session.appendMessage(userMsg("first question"));
const file = session.getSessionFile()!;
expect(readSessionFileRoles(file)).toEqual(["session", "model_change", "user"]);
expect(SessionManager.open(file, tempDir).buildSessionContext().messages).toHaveLength(1);
});
it("appends later entries to the file without rewriting earlier ones", () => {
const session = SessionManager.create(tempDir, tempDir);
session.appendMessage(userMsg("first question"));
session.appendCustomEntry("preset-state", { name: "plan" });
session.appendMessage(assistantMsg("first answer"));
expect(readSessionFileRoles(session.getSessionFile()!)).toEqual(["session", "user", "custom", "assistant"]);
});
});

View file

@ -1,9 +1,9 @@
import { existsSync, mkdirSync, readFileSync, rmSync } from "fs";
import { existsSync, mkdirSync, rmSync } from "fs";
import { tmpdir } from "os";
import { join } from "path";
import { describe, expect, it } from "vitest";
import { type CustomEntry, SessionManager } from "../../src/core/session-manager.ts";
import { assistantMsg, userMsg } from "../utilities.ts";
import { assistantMsg, readSessionFileRoles, userMsg } from "../utilities.ts";
describe("SessionManager append and tree traversal", () => {
describe("append operations", () => {
@ -480,45 +480,33 @@ describe("createBranchedSession", () => {
expect(entries.map((e) => e.id)).toEqual([id1, id2, id4, id5]);
});
it("does not duplicate entries when forking from first user message", () => {
it("does not duplicate entries when forking from before the first user message", () => {
const tempDir = join(tmpdir(), `session-fork-dedup-${Date.now()}`);
mkdirSync(tempDir, { recursive: true });
try {
// Create a persisted session with a couple of turns
const session = SessionManager.create(tempDir, tempDir);
const id1 = session.appendMessage(userMsg("first question"));
const modelChangeId = session.appendModelChange("anthropic", "claude-sonnet-4-5");
session.appendMessage(userMsg("first question"));
session.appendMessage(assistantMsg("first answer"));
session.appendMessage(userMsg("second question"));
session.appendMessage(assistantMsg("second answer"));
// Fork from the very first user message (no assistant in the branched path)
const newFile = session.createBranchedSession(id1);
// Fork from a setup entry (no user or assistant message in the branched path)
const newFile = session.createBranchedSession(modelChangeId);
expect(newFile).toBeDefined();
// The branched path has no assistant, so the file should not exist yet
// (deferred to _persist on first assistant, matching newSession() contract)
// Nothing to save yet, so the file is created later by the first user message
expect(existsSync(newFile!)).toBe(false);
session.appendMessage(userMsg("new question"));
expect(existsSync(newFile!)).toBe(true);
// Simulate extension adding entry before assistant (like preset on turn_start)
session.appendCustomEntry("preset-state", { name: "plan" });
// Now the assistant responds
session.appendMessage(assistantMsg("new answer"));
// File should now exist with exactly one header and no duplicate IDs
expect(existsSync(newFile!)).toBe(true);
const content = readFileSync(newFile!, "utf-8");
const lines = content.trim().split("\n").filter(Boolean);
const records = lines.map((line) => JSON.parse(line));
expect(records.filter((r) => r.type === "session")).toHaveLength(1);
const entryIds = records
.filter((r) => r.type !== "session")
.map((r) => r.id)
.filter((id): id is string => typeof id === "string");
expect(new Set(entryIds).size).toBe(entryIds.length);
// Exactly one header and each entry written once
expect(readSessionFileRoles(newFile!)).toEqual(["session", "model_change", "user", "custom", "assistant"]);
} finally {
rmSync(tempDir, { recursive: true, force: true });
}
@ -570,27 +558,20 @@ describe("createBranchedSession", () => {
}
});
it("writes file immediately when forking from a point with assistant messages", () => {
const tempDir = join(tmpdir(), `session-fork-with-assistant-${Date.now()}`);
it("writes file immediately when forking at a user message", () => {
const tempDir = join(tmpdir(), `session-fork-with-user-${Date.now()}`);
mkdirSync(tempDir, { recursive: true });
try {
const session = SessionManager.create(tempDir, tempDir);
session.appendMessage(userMsg("first question"));
const id2 = session.appendMessage(assistantMsg("first answer"));
session.appendMessage(userMsg("second question"));
session.appendMessage(assistantMsg("second answer"));
const id1 = session.appendMessage(userMsg("first question"));
session.appendMessage(assistantMsg("first answer"));
// Fork including the assistant message
const newFile = session.createBranchedSession(id2);
expect(newFile).toBeDefined();
// Path includes an assistant, so file should be written immediately
const newFile = session.createBranchedSession(id1);
expect(existsSync(newFile!)).toBe(true);
const content = readFileSync(newFile!, "utf-8");
const lines = content.trim().split("\n").filter(Boolean);
const records = lines.map((line) => JSON.parse(line));
expect(records.filter((r) => r.type === "session")).toHaveLength(1);
session.appendMessage(assistantMsg("new answer"));
expect(readSessionFileRoles(newFile!)).toEqual(["session", "user", "assistant"]);
} finally {
rmSync(tempDir, { recursive: true, force: true });
}

View file

@ -64,7 +64,7 @@ describe("SettingsSelectorComponent", () => {
modelThinkingLevels: {},
currentTheme: "dark",
terminalTheme: "dark",
availableThemes: ["dark", "light"],
availableThemes: ["system", "dark", "light"],
warnings: {},
} as unknown as SettingsConfig;
const callbacks = { onThemePreview: vi.fn(), onCancel: () => {} } as unknown as SettingsCallbacks;
@ -73,7 +73,9 @@ describe("SettingsSelectorComponent", () => {
list.selectItem("theme");
list.handleInput("\r");
let output = stripAnsi(list.render(120).join("\n"));
expect(output).toContain(" Automatic");
expect(output).toMatch(
/ {4}system +Theme created from your terminal's colors\n {4}automatic +Use separate themes/,
);
expect(output).toContain("→ ✓ dark");
list.handleInput("\x1b[B");

View file

@ -384,7 +384,7 @@ describe("AgentSessionRuntime characterization", () => {
expect(leafId).toBeTruthy();
await expect(runtime.fork(leafId!, { position: "at" })).rejects.toThrow(
"This session has not been saved yet. Wait for the first assistant response before cloning or forking it.",
"This session has not been saved yet. Send a message before cloning or forking it.",
);
});

View file

@ -1,6 +1,6 @@
import { resetCapabilitiesCache, setCapabilities } from "@earendil-works/pi-tui";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { highlightCode, initTheme } from "../src/modes/interactive/theme/theme.ts";
import { highlightCode, initTheme, theme } from "../src/modes/interactive/theme/theme.ts";
import {
highlight,
loadAllHighlightLanguages,
@ -99,15 +99,15 @@ describe("theme syntax highlighting", () => {
it("colors diff additions and deletions in fenced diff blocks", () => {
const lines = highlightCode("-old\n+new\n", "diff");
expect(lines[0]).toBe("\x1b[38;2;204;102;102m-old\x1b[39m");
expect(lines[1]).toBe("\x1b[38;2;181;189;104m+new\x1b[39m");
expect(lines[0]).toBe(`${theme.getFgAnsi("toolDiffRemoved")}-old\x1b[39m`);
expect(lines[1]).toBe(`${theme.getFgAnsi("toolDiffAdded")}+new\x1b[39m`);
});
it("keeps cli-highlight default styled scopes mapped to theme styles", () => {
expect(highlightCode("const re = /foo+/gi;", "javascript")[0]).toContain(
"\x1b[38;2;206;145;120m/foo+/gi\x1b[39m",
`${theme.getFgAnsi("syntaxString")}/foo+/gi\x1b[39m`,
);
expect(highlightCode("@decorator", "python")[0]).toBe("\x1b[38;2;128;128;128m@decorator\x1b[39m");
expect(highlightCode("<div></div>", "html")[0]).toContain("\x1b[38;2;86;156;214mdiv\x1b[39m");
expect(highlightCode("@decorator", "python")[0]).toBe(`${theme.getFgAnsi("muted")}@decorator\x1b[39m`);
expect(highlightCode("<div></div>", "html")[0]).toContain(`${theme.getFgAnsi("syntaxKeyword")}div\x1b[39m`);
});
});

View file

@ -0,0 +1,117 @@
import { colorToOklch, colorToRgb, parseColor, type RgbColor, rgbColor } from "@earendil-works/pi-tui";
import { afterEach, describe, expect, it } from "vitest";
import {
generateSystemThemeColors,
type SystemThemeInput,
wcagContrast,
} from "../src/modes/interactive/theme/system-theme.ts";
import {
getAvailableThemes,
getThemeByName,
getThemeExportColors,
setTerminalColors,
type ThemeToken,
} from "../src/modes/interactive/theme/theme.ts";
const rgb = (hex: string): RgbColor => colorToRgb(parseColor(hex));
const lightness = ({ r, g, b }: RgbColor) => colorToOklch(rgbColor(r, g, b)).l;
const DRACULA: SystemThemeInput = {
background: rgb("#282a36"),
foreground: rgb("#f8f8f2"),
palette: ["#21222c", "#ff5555", "#50fa7b", "#f1fa8c", "#bd93f9", "#ff79c6", "#8be9fd", "#f8f8f2"]
.concat(["#6272a4", "#ff6e6e", "#69ff94", "#ffffa5", "#d6acff", "#ff92df", "#a4ffff", "#ffffff"])
.map(rgb),
};
/** Dark with a palette, light with an unreadable foreground, background only, and mid-gray. */
const TERMINALS: Record<string, SystemThemeInput> = {
dracula: DRACULA,
solarizedLight: { background: rgb("#fdf6e3"), foreground: rgb("#657b83") },
backgroundOnly: { background: rgb("#1e1e1e") },
midGray: { background: rgb("#808080"), foreground: rgb("#ffffff") },
};
const PANELS: ThemeToken[] = ["userMessageBg", "toolPendingBg", "toolSuccessBg", "toolErrorBg", "selectedBg"];
function resolved(input: SystemThemeInput, token: ThemeToken): RgbColor {
const value = generateSystemThemeColors(input).colors[token];
if (value === "") return PANELS.includes(token) ? input.background! : input.foreground!;
return rgb(value as string);
}
afterEach(() => {
setTerminalColors({});
});
describe("generateSystemThemeColors", () => {
it("keeps body text readable (WCAG 4.5:1) on the background and its panels", () => {
for (const [name, input] of Object.entries(TERMINALS)) {
const text = resolved(input, "text");
for (const surface of [input.background!, resolved(input, "selectedBg")]) {
expect(wcagContrast(text, surface), name).toBeGreaterThanOrEqual(4.5);
}
expect(
wcagContrast(resolved(input, "toolTitle"), resolved(input, "toolErrorBg")),
name,
).toBeGreaterThanOrEqual(4.5);
}
});
it("orders foreground roles by contrast and keeps panels close to the background", () => {
for (const [name, input] of Object.entries(TERMINALS)) {
const background = lightness(input.background!);
const offset = (token: ThemeToken) => lightness(resolved(input, token)) - background;
// On mid-gray the levels collapse to the strongest reachable color.
if (name !== "midGray") {
expect(Math.abs(offset("text")), name).toBeGreaterThan(Math.abs(offset("muted")));
expect(Math.abs(offset("muted")), name).toBeGreaterThan(Math.abs(offset("dim")));
}
const lighter = generateSystemThemeColors(input).appearance === "dark";
for (const panel of PANELS) {
expect(wcagContrast(resolved(input, panel), input.background!), `${name} ${panel}`).toBeLessThan(2);
expect(offset(panel) > 0, `${name} ${panel}`).toBe(lighter);
}
}
});
it("uses the terminal foreground and palette hues", () => {
expect(generateSystemThemeColors(DRACULA).colors.text).toBe("");
// Solarized's foreground is below 4.5:1 on its own background, so text is darkened.
expect(generateSystemThemeColors(TERMINALS.solarizedLight).colors.text).not.toBe("");
const hue = ({ r, g, b }: RgbColor) => colorToOklch(rgbColor(r, g, b)).h;
expect(Math.abs(hue(resolved(DRACULA, "error")) - hue(DRACULA.palette![1]))).toBeLessThan(8);
});
it("renders grayscale at zero saturation", () => {
const { colors } = generateSystemThemeColors({ ...DRACULA, saturation: 0 });
expect(colorToOklch(parseColor(colors.error as string)).c).toBeLessThan(0.005);
});
it("falls back to palette indices and faint text without a background", () => {
const { colors, dim, appearance } = generateSystemThemeColors({ appearanceHint: "light" });
expect(appearance).toBe("light");
expect([colors.error, colors.text, colors.userMessageBg]).toEqual([1, "", ""]);
expect(dim).toContain("muted");
expect(generateSystemThemeColors({ saturation: 0 }).colors.error).toBe("");
});
});
describe("system theme", () => {
it("is listed first, has no export colors, and is generated from the terminal colors", () => {
expect(getAvailableThemes()[0]).toBe("system");
expect(getThemeExportColors("system")).toEqual({});
setTerminalColors(DRACULA);
const theme = getThemeByName("system")!;
expect(theme.appearance).toBe("dark");
expect(theme.getFgAnsi("text")).toBe("\x1b[39m");
expect(theme.getFgAnsi("error")).toMatch(/^\x1b\[38;/);
});
it("renders faint tokens with SGR 2 and closes it", () => {
const theme = getThemeByName("system")!;
expect(theme.fg("muted", "x")).toBe("\x1b[39m\x1b[2mx\x1b[22;39m");
expect(theme.style("x", { fg: "muted" })).toBe("\x1b[39m\x1b[2mx\x1b[22m\x1b[39m");
});
});

View file

@ -1,18 +1,22 @@
import type { TUI } from "@earendil-works/pi-tui";
import type { TerminalColors, TUI } from "@earendil-works/pi-tui";
import { afterEach, describe, expect, it, vi } from "vitest";
import { SettingsManager } from "../src/core/settings-manager.ts";
import {
initTheme,
setTerminalDefaultColors,
setTerminalColorScheme,
setTerminalColors,
type TerminalTheme,
theme,
} from "../src/modes/interactive/theme/theme.ts";
import { InteractiveThemeController } from "../src/modes/interactive/theme/theme-controller.ts";
const DARK: TerminalColors = { foreground: { r: 248, g: 248, b: 242 }, background: { r: 40, g: 42, b: 54 } };
const LIGHT: TerminalColors = { foreground: { r: 30, g: 30, b: 30 }, background: { r: 250, g: 250, b: 250 } };
type ColorQueryOptions = { timeoutMs: number; onLateReply?: (colors: TerminalColors) => void };
function createUi() {
const queryTerminalBackgroundColor = vi.fn();
const queryTerminalForegroundColor = vi.fn();
const queryTerminalColorScheme = vi.fn();
const queryTerminalColors = vi.fn(async (_options: ColorQueryOptions): Promise<TerminalColors> => ({}));
const setTerminalColorSchemeNotifications = vi.fn();
let terminalColorSchemeListener: ((terminalTheme: TerminalTheme) => void) | undefined;
const unsubscribeTerminalColorScheme = vi.fn();
@ -24,15 +28,11 @@ function createUi() {
terminalColorSchemeListener = listener;
return unsubscribeTerminalColorScheme;
}),
queryTerminalBackgroundColor,
queryTerminalForegroundColor,
queryTerminalColorScheme,
queryTerminalColors,
} as unknown as TUI;
return {
ui,
queryTerminalBackgroundColor,
queryTerminalForegroundColor,
queryTerminalColorScheme,
queryTerminalColors,
setTerminalColorSchemeNotifications,
unsubscribeTerminalColorScheme,
emitTerminalColorScheme: (terminalTheme: TerminalTheme) => terminalColorSchemeListener?.(terminalTheme),
@ -48,79 +48,119 @@ function createController(ui: TUI, getSettingsManager: () => SettingsManager, in
});
}
const flush = () => new Promise((resolve) => setTimeout(resolve, 0));
afterEach(() => {
setTerminalColors({});
setTerminalColorScheme(undefined);
initTheme("dark");
setTerminalDefaultColors({});
vi.unstubAllEnvs();
});
describe("InteractiveThemeController", () => {
it("uses the initial theme without persisting it", async () => {
const { ui, queryTerminalBackgroundColor } = createUi();
const { ui, queryTerminalColors } = createUi();
const manager = SettingsManager.inMemory({ theme: "dark" });
const setTheme = vi.spyOn(manager, "setTheme");
const flush = vi.spyOn(manager, "flush");
const flushSettings = vi.spyOn(manager, "flush");
const controller = createController(ui, () => manager, "light");
expect(theme.name).toBe("light");
expect(controller.getThemeSelection()).toBe("light");
await controller.applyFromSettings();
controller.applyFromSettings();
await flush();
expect(queryTerminalBackgroundColor).toHaveBeenCalledOnce();
expect(queryTerminalColors).toHaveBeenCalledOnce();
expect(setTheme).not.toHaveBeenCalled();
expect(flush).not.toHaveBeenCalled();
expect(flushSettings).not.toHaveBeenCalled();
});
it("resolves a theme pair and follows terminal appearance changes", async () => {
vi.stubEnv("COLORFGBG", "15;0");
const { ui, queryTerminalColorScheme, setTerminalColorSchemeNotifications, emitTerminalColorScheme } = createUi();
queryTerminalColorScheme.mockResolvedValue("light");
const manager = SettingsManager.inMemory({ theme: "dark/light" });
const controller = createController(ui, () => manager, "light/dark");
it("applies the theme immediately and lets startup wait for the colors", async () => {
const { ui, queryTerminalColors } = createUi();
let answer: (colors: TerminalColors) => void = () => {};
queryTerminalColors.mockReturnValue(
new Promise((resolve) => {
answer = resolve;
}),
);
const controller = createController(ui, () => SettingsManager.inMemory());
controller.applyFromSettings();
expect(theme.name).toBe("dark");
await controller.applyFromSettings();
expect(theme.name).toBe("light");
// Grayscale until the terminal answers.
expect(theme.name).toBe("system");
expect(theme.getFgAnsi("error")).toBe("\x1b[39m");
answer(DARK);
await controller.waitForTerminalColors();
expect(theme.getFgAnsi("error")).toMatch(/^\x1b\[38;/);
});
it("falls back to palette indices, then applies colors that arrive after the timeout", async () => {
const { ui, queryTerminalColors } = createUi();
let lateReply: (colors: TerminalColors) => void = () => {};
queryTerminalColors.mockImplementation(async (options) => {
lateReply = options.onLateReply!;
return {};
});
const controller = createController(ui, () => SettingsManager.inMemory());
controller.applyFromSettings();
await flush();
expect(theme.getFgAnsi("error")).toBe("\x1b[38;5;1m");
lateReply(DARK);
expect(theme.colors.error.kind).toBe("rgb");
});
it("re-queries the colors on appearance changes and lets them decide", async () => {
const { ui, queryTerminalColors, setTerminalColorSchemeNotifications, emitTerminalColorScheme } = createUi();
queryTerminalColors.mockResolvedValue(LIGHT);
const controller = createController(ui, () => SettingsManager.inMemory(), "light/dark");
controller.applyFromSettings();
expect(setTerminalColorSchemeNotifications).toHaveBeenCalledWith(true);
await flush();
expect(theme.name).toBe("light");
emitTerminalColorScheme("dark");
queryTerminalColors.mockResolvedValue(DARK);
// The report says light, but the terminal renders dark.
emitTerminalColorScheme("light");
await flush();
expect(theme.name).toBe("dark");
});
it("detects the theme from the default color query without querying twice", async () => {
it("uses the reported scheme for the system theme when the terminal reports no colors", async () => {
vi.stubEnv("COLORFGBG", "");
const { ui, queryTerminalBackgroundColor } = createUi();
queryTerminalBackgroundColor.mockResolvedValue({ r: 250, g: 250, b: 250 });
await createController(ui, () => SettingsManager.inMemory()).applyFromSettings();
const { ui, emitTerminalColorScheme } = createUi();
const controller = createController(ui, () => SettingsManager.inMemory());
controller.applyFromSettings();
await flush();
expect(theme.appearance).toBe("dark");
expect(theme.name).toBe("light");
expect(queryTerminalBackgroundColor).toHaveBeenCalledOnce();
emitTerminalColorScheme("light");
expect(theme.appearance).toBe("light");
expect(controller.getTerminalTheme()).toBe("light");
});
it("re-renders only when the reported default colors change", async () => {
const { ui, queryTerminalForegroundColor, queryTerminalBackgroundColor } = createUi();
it("re-renders only when the reported colors change", async () => {
const { ui, queryTerminalColors } = createUi();
const controller = createController(ui, () => SettingsManager.inMemory({ theme: "dark" }));
const query = async (foreground?: object, background?: object) => {
queryTerminalForegroundColor.mockResolvedValue(foreground);
queryTerminalBackgroundColor.mockResolvedValue(background);
await controller.applyFromSettings();
await new Promise((resolve) => setTimeout(resolve, 0));
const query = async (colors: TerminalColors) => {
queryTerminalColors.mockResolvedValue(colors);
controller.applyFromSettings();
await flush();
};
await query({ r: 200, g: 210, b: 220 }, { r: 10, g: 20, b: 30 });
await query(DARK);
// A timeout keeps the known colors; erasing them would count as a change and re-render.
await query(undefined, undefined);
await query({ r: 200, g: 210, b: 220 }, { r: 10, g: 20, b: 30 });
await query({});
await query(structuredClone(DARK));
expect(ui.requestRender).toHaveBeenCalledOnce();
});
it("disables terminal appearance updates when disposed", async () => {
const { ui, queryTerminalColorScheme, setTerminalColorSchemeNotifications, unsubscribeTerminalColorScheme } =
createUi();
queryTerminalColorScheme.mockResolvedValue("light");
const manager = SettingsManager.inMemory({ theme: "light/dark" });
const controller = createController(ui, () => manager);
await controller.applyFromSettings();
const { ui, setTerminalColorSchemeNotifications, unsubscribeTerminalColorScheme } = createUi();
const controller = createController(ui, () => SettingsManager.inMemory({ theme: "light/dark" }));
controller.applyFromSettings();
await flush();
controller.dispose();
@ -128,30 +168,18 @@ describe("InteractiveThemeController", () => {
expect(unsubscribeTerminalColorScheme).toHaveBeenCalledOnce();
});
it("detects the current terminal appearance when selecting a theme pair", async () => {
vi.stubEnv("COLORFGBG", "");
const { ui, queryTerminalColorScheme } = createUi();
queryTerminalColorScheme.mockResolvedValue("light");
const manager = SettingsManager.inMemory({ theme: "dark" });
const controller = createController(ui, () => manager);
expect(theme.name).toBe("dark");
await controller.setThemeSetting("light/dark");
expect(theme.name).toBe("light");
expect(queryTerminalColorScheme).toHaveBeenCalledOnce();
});
it("lets an explicit selection replace the initial theme", async () => {
const { ui } = createUi();
const firstManager = SettingsManager.inMemory({ theme: "dark" });
const secondManager = SettingsManager.inMemory({ theme: "light" });
let manager = firstManager;
const controller = createController(ui, () => manager, "light");
await controller.applyFromSettings();
controller.applyFromSettings();
expect(controller.setThemeName("dark")).toEqual({ success: true });
manager = secondManager;
await controller.applyFromSettings();
controller.applyFromSettings();
await flush();
expect(controller.getThemeSelection()).toBe("dark");
expect(theme.name).toBe("dark");
@ -163,15 +191,15 @@ describe("InteractiveThemeController", () => {
const secondManager = SettingsManager.inMemory({ theme: "light" });
let manager = firstManager;
const controller = createController(ui, () => manager);
await controller.applyFromSettings();
controller.applyFromSettings();
firstManager.applyOverrides({ theme: "light" });
await controller.applyFromSettings();
controller.applyFromSettings();
expect(theme.name).toBe("light");
secondManager.applyOverrides({ theme: "dark" });
manager = secondManager;
await controller.applyFromSettings();
controller.applyFromSettings();
expect(theme.name).toBe("dark");
});
});

View file

@ -1,11 +1,9 @@
import { type RgbColor, resetCapabilitiesCache, setCapabilities } from "@earendil-works/pi-tui";
import { resetCapabilitiesCache, setCapabilities } from "@earendil-works/pi-tui";
import { afterEach, describe, expect, it } from "vitest";
import {
detectTerminalBackgroundFromEnv,
detectTerminalBackgroundTheme,
detectTerminalThemeForAuto,
detectColorFgBgTheme,
detectTerminalTheme,
getThemeByName,
getThemeForRgbColor,
parseAutoThemeSetting,
resolveThemeSetting,
} from "../src/modes/interactive/theme/theme.ts";
@ -14,129 +12,34 @@ afterEach(() => {
resetCapabilitiesCache();
});
describe("detectTerminalBackgroundFromEnv", () => {
it("uses the COLORFGBG background color index", () => {
expect(detectTerminalBackgroundFromEnv({ env: { COLORFGBG: "0;15" } })).toMatchObject({
theme: "light",
source: "COLORFGBG",
confidence: "high",
});
expect(detectTerminalBackgroundFromEnv({ env: { COLORFGBG: "15;0" } })).toMatchObject({
theme: "dark",
source: "COLORFGBG",
confidence: "high",
});
});
it("uses the last COLORFGBG field as the background", () => {
expect(detectTerminalBackgroundFromEnv({ env: { COLORFGBG: "0;7;15" } }).theme).toBe("light");
});
it("defaults to dark without terminal background hints", () => {
expect(detectTerminalBackgroundFromEnv({ env: {} })).toMatchObject({
theme: "dark",
source: "fallback",
confidence: "low",
});
describe("detectColorFgBgTheme", () => {
it("classifies the last field by palette index like Vim", () => {
expect(detectColorFgBgTheme({ COLORFGBG: "15;0" })).toBe("dark");
expect(detectColorFgBgTheme({ COLORFGBG: "0;7;15" })).toBe("light");
// Solarized Dark's background is bright black.
expect(detectColorFgBgTheme({ COLORFGBG: "12;8" })).toBe("dark");
// rxvt writes "default" when the background is not a palette color.
expect(detectColorFgBgTheme({ COLORFGBG: "15;default" })).toBeUndefined();
expect(detectColorFgBgTheme({})).toBeUndefined();
});
});
describe("detectTerminalBackgroundTheme", () => {
it("uses the queried terminal background before environment hints", async () => {
let queriedTimeoutMs: number | undefined;
const detection = await detectTerminalBackgroundTheme({
env: { COLORFGBG: "15;0" },
timeoutMs: 250,
ui: {
async queryTerminalBackgroundColor({ timeoutMs }: { timeoutMs: number }): Promise<RgbColor | undefined> {
queriedTimeoutMs = timeoutMs;
return { r: 250, g: 250, b: 250 };
},
},
});
expect(queriedTimeoutMs).toBe(250);
expect(detection).toMatchObject({
theme: "light",
source: "terminal background",
confidence: "high",
});
describe("detectTerminalTheme", () => {
it("prefers the background, then the reported scheme, then COLORFGBG, then dark", () => {
const env = { COLORFGBG: "0;15" };
expect(detectTerminalTheme({ background: { r: 8, g: 8, b: 8 } }, "light", env)).toBe("dark");
expect(detectTerminalTheme({}, "dark", env)).toBe("dark");
expect(detectTerminalTheme({}, undefined, env)).toBe("light");
expect(detectTerminalTheme({}, undefined, {})).toBe("dark");
});
it("falls back to environment hints when the terminal query returns no color", async () => {
const detection = await detectTerminalBackgroundTheme({
env: { COLORFGBG: "15;0" },
timeoutMs: 250,
ui: {
async queryTerminalBackgroundColor(): Promise<RgbColor | undefined> {
return undefined;
},
},
});
expect(detection).toMatchObject({
theme: "dark",
source: "COLORFGBG",
confidence: "high",
});
});
it("falls back to environment hints when the terminal query fails", async () => {
const detection = await detectTerminalBackgroundTheme({
env: { COLORFGBG: "0;15" },
timeoutMs: 250,
ui: {
async queryTerminalBackgroundColor(): Promise<RgbColor | undefined> {
throw new Error("terminal write failed");
},
},
});
expect(detection).toMatchObject({
theme: "light",
source: "COLORFGBG",
confidence: "high",
});
});
});
describe("detectTerminalThemeForAuto", () => {
it("starts both queries and returns the preferred color-scheme result without waiting", async () => {
let resolveColorScheme!: (theme: "dark" | "light" | undefined) => void;
let backgroundQueryStarted = false;
const detection = detectTerminalThemeForAuto({
timeoutMs: 100,
ui: {
queryTerminalColorScheme: () =>
new Promise((resolve) => {
resolveColorScheme = resolve;
}),
queryTerminalBackgroundColor: () => {
backgroundQueryStarted = true;
return new Promise<RgbColor | undefined>(() => {});
},
},
});
expect(backgroundQueryStarted).toBe(true);
resolveColorScheme("dark");
await expect(detection).resolves.toBe("dark");
});
it("uses the background result when the color-scheme query fails", async () => {
await expect(
detectTerminalThemeForAuto({
timeoutMs: 100,
ui: {
async queryTerminalColorScheme(): Promise<undefined> {
throw new Error("color-scheme query failed");
},
async queryTerminalBackgroundColor(): Promise<RgbColor> {
return { r: 250, g: 250, b: 250 };
},
},
}),
).resolves.toBe("light");
it("follows the foreground when text is readable that way", () => {
const background = { r: 118, g: 118, b: 118 };
expect(detectTerminalTheme({ background })).toBe("light");
expect(detectTerminalTheme({ background, foreground: { r: 255, g: 255, b: 255 } })).toBe("dark");
// White text cannot reach 4.5:1 on mid-gray.
const midGray = { r: 128, g: 128, b: 128 };
expect(detectTerminalTheme({ background: midGray, foreground: { r: 255, g: 255, b: 255 } })).toBe("light");
});
});
@ -156,13 +59,6 @@ describe("theme color mode", () => {
});
});
describe("theme detection from RGB", () => {
it("classifies RGB colors by luminance", () => {
expect(getThemeForRgbColor({ r: 8, g: 8, b: 8 })).toBe("dark");
expect(getThemeForRgbColor({ r: 250, g: 250, b: 250 })).toBe("light");
});
});
describe("theme setting helpers", () => {
it("parses and resolves automatic theme settings", () => {
expect(parseAutoThemeSetting("light/dark")).toEqual({ lightTheme: "light", darkTheme: "dark" });

View file

@ -1,3 +1,4 @@
import { colorToHex, okhslColor } from "@earendil-works/pi-tui";
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "fs";
import { tmpdir } from "os";
import { join } from "path";
@ -69,6 +70,28 @@ describe("getThemeExportColors", () => {
});
});
it("converts OKHSL export colors to hex because CSS does not support them", () => {
const darkTheme = JSON.parse(
readFileSync(new URL("../src/modes/interactive/theme/dark.json", import.meta.url), "utf-8"),
) as ThemeFile;
const customTheme: ThemeFile = {
...darkTheme,
name: "custom-export-okhsl",
vars: { card: "okhsl(250 20% 20%)" },
export: { pageBg: "okhsl(250 20% 15%)", cardBg: "card", infoBg: "oklch(30% 0.05 80)" },
};
writeFileSync(
join(process.env.PI_CODING_AGENT_DIR!, "themes", "custom-export-okhsl.json"),
JSON.stringify(customTheme, null, 2),
);
expect(getThemeExportColors("custom-export-okhsl")).toEqual({
pageBg: colorToHex(okhslColor(250, 0.2, 0.15)),
cardBg: colorToHex(okhslColor(250, 0.2, 0.2)),
infoBg: "oklch(30% 0.05 80)",
});
});
it("resolves recursive vars and converts 256-color export values to hex", () => {
const darkTheme = JSON.parse(
readFileSync(new URL("../src/modes/interactive/theme/dark.json", import.meta.url), "utf-8"),

View file

@ -1,13 +1,18 @@
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { colorToHex, styleText } from "@earendil-works/pi-tui";
import { colorToHex, okhslColor, styleText } from "@earendil-works/pi-tui";
import { afterEach, describe, expect, it } from "vitest";
import { loadThemeFromPath, setTerminalDefaultColors } from "../src/modes/interactive/theme/theme.ts";
import { loadThemeFromPath, setTerminalColors } from "../src/modes/interactive/theme/theme.ts";
const tempDirs: string[] = [];
type ThemeFile = { name: string; appearance?: "dark" | "light"; colors: Record<string, string | number> };
type ThemeFile = {
name: string;
appearance?: "dark" | "light";
vars?: Record<string, string | number>;
colors: Record<string, string | number>;
};
/** Load a copy of a built-in theme, modified by `edit`. */
function loadTheme(base: "dark" | "light", edit: (theme: ThemeFile) => void = () => {}) {
@ -23,7 +28,7 @@ function loadTheme(base: "dark" | "light", edit: (theme: ThemeFile) => void = ()
}
afterEach(() => {
setTerminalDefaultColors({});
setTerminalColors({});
for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
});
@ -49,9 +54,23 @@ describe("theme styles", () => {
expect(theme.colors.accent).toEqual({ kind: "oklch", l: 0.62, c: 0.1, h: 200 });
});
it("loads OKHSL theme values, including through variables", () => {
const theme = loadTheme("dark", (json) => {
json.vars = { ...json.vars, brand: "okhsl(250 60% 55%)" };
json.colors.accent = "brand";
json.colors.error = "okhsl(20 90% 60%)";
});
expect(colorToHex(theme.colors.accent)).toBe(colorToHex(okhslColor(250, 0.6, 0.55)));
expect(colorToHex(theme.colors.error)).toBe(colorToHex(okhslColor(20, 0.9, 0.6)));
});
it("detects the appearance unless it is declared", () => {
expect(loadTheme("dark").appearance).toBe("dark");
expect(loadTheme("light").appearance).toBe("light");
// Without a declaration, the appearance is detected from the theme's own colors.
for (const base of ["dark", "light"] as const) {
expect(loadTheme(base, (json) => delete json.appearance).appearance).toBe(base);
}
expect(
loadTheme("dark", (json) => {
json.appearance = "light";
@ -60,10 +79,11 @@ describe("theme styles", () => {
// Palette colors 0-15 follow the terminal palette, so such themes follow the terminal background.
const paletteOnly = loadTheme("dark", (json) => {
delete json.appearance;
for (const key of Object.keys(json.colors)) json.colors[key] = key.endsWith("Bg") ? 0 : 7;
});
expect(paletteOnly.appearance).toBe("dark");
setTerminalDefaultColors({ background: { r: 250, g: 250, b: 250 } });
setTerminalColors({ background: { r: 250, g: 250, b: 250 } });
expect(paletteOnly.appearance).toBe("light");
});
@ -77,7 +97,7 @@ describe("theme styles", () => {
expect(colorToHex(theme.colors.text)).toBe("#e5e5e7");
expect(colorToHex(theme.colors.userMessageBg)).toBe("#000000");
setTerminalDefaultColors({ foreground: { r: 200, g: 210, b: 220 }, background: { r: 10, g: 20, b: 30 } });
setTerminalColors({ foreground: { r: 200, g: 210, b: 220 }, background: { r: 10, g: 20, b: 30 } });
expect(colorToHex(theme.colors.text)).toBe("#c8d2dc");
expect(colorToHex(theme.colors.userMessageBg)).toBe("#0a141e");
});

View file

@ -0,0 +1,26 @@
import { afterEach, describe, expect, it } from "vitest";
import { ThemedText } from "../src/modes/interactive/components/themed-text.ts";
import { initTheme, theme } from "../src/modes/interactive/theme/theme.ts";
afterEach(() => {
initTheme("dark");
});
describe("ThemedText", () => {
it("builds lazily and rebuilds with the current theme after invalidation", () => {
initTheme("dark");
let builds = 0;
const text = new ThemedText(() => {
builds++;
return theme.fg("accent", "hello");
});
expect(builds).toBe(0);
const dark = text.render(20).join("");
initTheme("light");
expect(text.render(20).join("")).toBe(dark);
text.invalidate();
expect(text.render(20).join("")).toContain(theme.getFgAnsi("accent"));
expect(builds).toBe(2);
});
});

View file

@ -152,6 +152,20 @@ export function assistantMsg(text: string) {
};
}
/**
* Read a session JSONL file and return one label per record: the message role for
* message entries, otherwise the entry type (e.g. "session", "model_change").
*/
export function readSessionFileRoles(file: string): string[] {
return readFileSync(file, "utf-8")
.trim()
.split("\n")
.map((line) => {
const record = JSON.parse(line);
return record.message?.role ?? record.type;
});
}
/**
* Options for creating a test session.
*/

View file

@ -6,11 +6,17 @@
- Reordered Storage scan arguments so the limit precedes the cursor.
- Added the required conversation-visible `Storage.entry(conversationId, id, context)` overload.
- Split `Tx.createConversation()` from `Tx.forkConversation()`, replaced raw conversation-record input, and require explicit ownerless or task ownership.
- Replaced untyped numeric record IDs and the `TaskRef` wrapper with erased branded numeric ID types, including result-typed `TaskId<R>`, separately branded commit sequences, and generic `Storage.mintId()`.
- Made task conversation membership immutable after task creation.
- Added `ConversationQuery` to Storage and transaction conversation scans.
### Added
- Added transactional Sessions with typed durable documents, task creation, snapshots, retirement, and commit publications.
- Added document checkpoint selection, lazy version migration, and `Session.snapshotAsOf()` for rewindable conversation documents.
- Added policy-driven backend-side conversation document copying when creating forks.
- Added indexed conversation ownership queries and guaranteed no-effect Storage rejection handling.
## [0.87.1] - 2026-09-22

View file

@ -70,20 +70,37 @@ The concrete JSON representations may add bookkeeping fields, but must preserve
these contracts.
```ts
type Id = number;
declare const idBrand: unique symbol;
type Id<Kind extends string, Type = unknown> = number & {
readonly [idBrand]: {
readonly kind: Kind;
readonly type: Type;
};
};
type ConversationId = Id<"conversation">;
type EntryId = Id<"entry">;
type TaskId<Result = unknown> = Id<"task", Result>;
type SubmissionId = Id<"submission">;
type DocumentId = Id<"document">;
declare const seqBrand: unique symbol;
/** Strictly increases between commits; gaps are permitted. */
type Seq = number;
const ROOT_CONVERSATION_ID: Id = 1;
type Seq = number & { readonly [seqBrand]: "sequence" };
const ROOT_CONVERSATION_ID = 1 as ConversationId;
type ConversationOwnership =
| { readonly kind: "ownerless" }
| { readonly kind: "task"; readonly taskId: TaskId };
type ConversationRecord = {
readonly id: Id;
readonly id: ConversationId;
readonly parent?: {
readonly conversationId: Id;
readonly at: Id;
readonly conversationId: ConversationId;
readonly at: EntryId;
};
readonly owner?: {
readonly conversationId: Id;
readonly taskId: Id;
readonly conversationId: ConversationId;
readonly taskId: TaskId;
};
};
```
@ -109,30 +126,30 @@ order yields the effective prompt and tool set.
```ts
type ContextEdit = {
readonly target: Id;
readonly target: EntryId;
} & (
| { readonly action: "omit"; readonly messages?: never }
| { readonly action: "replace"; readonly messages: readonly Message[] }
);
type EntryRecord = {
readonly id: Id;
readonly conversationId: Id;
readonly id: EntryId;
readonly conversationId: ConversationId;
readonly kind: string;
readonly model?: readonly Message[];
readonly data?: JsonValue;
readonly head?: Id;
readonly head?: EntryId;
readonly edits?: readonly ContextEdit[];
readonly byTaskId?: Id;
readonly byTaskId?: TaskId;
};
type EntryDraft = Omit<EntryRecord, "id" | "conversationId" | "byTaskId" | "head"> & {
readonly head?: Id | "self";
readonly head?: EntryId | "self";
};
type SubmissionRecordBase = {
readonly id: Id;
readonly conversationId: Id;
readonly id: SubmissionId;
readonly conversationId: ConversationId;
readonly requestId?: string;
};
@ -149,21 +166,21 @@ type SubmissionRecord =
}
| {
readonly status: "placed";
readonly entry: Id;
readonly entry: EntryId;
readonly answer?: never;
readonly reason?: never;
readonly detail?: never;
}
| {
readonly status: "done";
readonly entry: Id;
readonly answer: Id;
readonly entry: EntryId;
readonly answer: EntryId;
readonly reason?: never;
readonly detail?: never;
}
| {
readonly status: "unanswered";
readonly entry?: Id;
readonly entry?: EntryId;
readonly answer?: never;
readonly reason: string;
readonly detail?: JsonValue;
@ -181,7 +198,7 @@ type SubmissionRecord =
}
| {
readonly status: "done";
readonly entry: Id;
readonly entry: EntryId;
readonly answer?: never;
readonly reason?: never;
readonly detail?: never;
@ -202,10 +219,18 @@ type SubmissionCreate = SubmissionRecord extends infer Record
: never;
```
ID brands are compile-time only. IDs and sequences remain ordinary numbers in
memory, JSON, JSONL, and SQLite. Code applies a brand only at a trusted creation
or decoding boundary. The distinct sequence brand prevents an entity ID from
being used as a document commit point.
Conversation history parenting and task ownership are separate:
- `parent` controls inherited entries and historical documents.
- `owner` controls task authorization, subtree abort, and subtree idle waits.
- `owner` records task attribution and connects scopes for subtree abort and idle traversal. It is not an access-control capability.
- Creation always supplies `ConversationOwnership` explicitly. The Session
derives the persisted owner's conversation from the named task; callers never
construct the persisted owner pair.
A conversation's owner remains recorded after the owning task becomes terminal.
@ -277,12 +302,15 @@ type SectionSeed =
| { readonly key: string; readonly remove: true };
type ConversationSpec = {
readonly parent?: { readonly conversationId: Id; readonly at: Id };
readonly model?: ModelRef;
readonly sections?: readonly SectionSeed[];
readonly activeTools?: readonly string[];
};
type ConversationCreateSpec = ConversationSpec & {
readonly ownership: ConversationOwnership;
};
type AnyTask = {
readonly definition: {
readonly name: string;
@ -301,7 +329,7 @@ type HarnessOptions = {
readonly taskKinds?: readonly AnyTask[];
readonly sections?: readonly SystemSection<JsonValue>[];
readonly now?: () => number;
readonly root?: Omit<ConversationSpec, "parent">;
readonly root?: ConversationSpec;
readonly onReport?: (error: unknown) => void;
};
@ -323,7 +351,7 @@ type SettledSubmissionRecord = SubmissionRecord & {
};
interface Submission {
readonly id: Id;
readonly id: SubmissionId;
status(context: Context): Promise<SubmissionRecord>;
wait(context: Context): Promise<SettledSubmissionRecord>;
abort(context: Context): Promise<"aborted" | "already_placed" | "settled">;
@ -340,7 +368,7 @@ type HooksOf<K> = K extends Task<infer _I, infer _S, infer _R, infer H>
: never;
interface Conversation {
readonly id: Id;
readonly id: ConversationId;
submit(submission: SubmissionDraft, context: Context): Promise<Submission>;
getModel(context: Context): Promise<ModelRef | undefined>;
@ -371,11 +399,11 @@ interface Conversation {
context: Context,
): Promise<Page<EntryRecord, Cursor>>;
fork(
at: Id,
spec: Omit<ConversationSpec, "parent">,
at: EntryId,
spec: ConversationCreateSpec,
context: Context,
): Promise<Conversation>;
collapse(instructions: string | undefined, context: Context): Promise<Id>;
collapse(instructions: string | undefined, context: Context): Promise<TaskId>;
reset(handoff: string | undefined, context: Context): Promise<void>;
abort(context: Context): Promise<void>;
waitForIdle(context: Context): Promise<void>;
@ -405,22 +433,22 @@ interface Harness extends Session {
root(context: Context): Promise<Conversation>;
onConversation(listener: (conversation: Conversation) => void): () => void;
conversation(id: Id, context: Context): Promise<Conversation | undefined>;
conversation(id: ConversationId, context: Context): Promise<Conversation | undefined>;
createConversation(
spec: ConversationSpec & { readonly input?: UserInput },
spec: ConversationCreateSpec & { readonly input?: UserInput },
context: Context,
): Promise<Conversation>;
getTask(id: Id, context: Context): Promise<TaskRecord<JsonValue, JsonValue, JsonValue> | undefined>;
submission(id: Id, context: Context): Promise<Submission | undefined>;
getTask<R>(id: TaskId<R>, context: Context): Promise<TaskRecord<JsonValue, JsonValue, R> | undefined>;
submission(id: SubmissionId, context: Context): Promise<Submission | undefined>;
abortSubmission(
id: Id,
id: SubmissionId,
context: Context,
conversationId?: Id,
conversationId?: ConversationId,
): Promise<"aborted" | "already_placed" | "settled" | "not_found">;
abortTask(id: Id, context: Context): Promise<"marked" | "terminal">;
markTask(id: Id, context: Context): Promise<"marked" | "terminal">;
waitForTask<R>(ref: TaskRef<R>, context: Context): Promise<SettledTask<R>>;
abortTask(id: TaskId, context: Context): Promise<"marked" | "terminal">;
markTask(id: TaskId, context: Context): Promise<"marked" | "terminal">;
waitForTask<R>(id: TaskId<R>, context: Context): Promise<SettledTask<R>>;
waitForIdle(context: Context): Promise<void>;
}
@ -450,8 +478,8 @@ handler dispatches during open. Dynamic registration is available after open and
does not resurrect a task already settled by that pass.
The root conversation always has reserved ID `ROOT_CONVERSATION_ID` (`1`). Empty
storage creates that conversation from `options.root`; reopen looks it up by the
reserved ID. `options.root` never overwrites existing state. `root()` returns
storage creates that conversation from `options.root` as ownerless; reopen looks
it up by the reserved ID. `options.root` never overwrites existing state. `root()` returns
that handle. A conversation with no configured model produces a durable
`no_model` generation failure.
@ -467,11 +495,14 @@ function that unregisters only that exact token. It supports declaration
movement during normal operation, not replacement of executing extension code;
section 7.4 governs code reload.
Conversation creation atomically commits the conversation, built-in
configuration and section/tool seeds, and an optional input submission. For a fork,
omitted model, section, and active-tool values follow their document definition's
fork policy; provided values override those forked built-in values in the same
commit. Other documents follow their own definitions without special handling.
Conversation creation atomically commits the conversation, its explicitly
selected ownership, built-in configuration and section/tool seeds, and an
optional input submission. Host callers must choose ownerless or task ownership
in `ConversationCreateSpec`; neither the Harness nor a conversation handle
infers ownership from call context. For a fork, omitted model, section, and
active-tool values follow their document definition's fork policy; provided
values override those forked built-in values in the same commit. Other documents
follow their own definitions without special handling.
The built-in conversation configuration document contains the selected model,
thinking level, an ordered array of section key/value records, and active tool
@ -518,23 +549,25 @@ fork-aware stored history; callers cannot substitute another conversation ID.
Generic Session-wide document operations remain available directly on `Harness`
because `Harness extends Session`.
`fork()` requires a concrete visible parent entry and applies section 3.7.
`fork()` requires a concrete visible parent entry and explicit ownership, then
applies section 3.7.
`collapse()` returns the newly admitted background collapse task ID, not its
future summary entry. `reset()` durably admits a passive self-head reset or
handoff write and then resolves; while busy, placement follows section 6 and may
occur later. Observe its placement through the conversation watch. An idle wait
does not guarantee placement of queued passive writes.
`markTask()` only commits `abortRequested`; it neither signals nor joins an
active invocation. The scheduler notices the mark on its next drain.
`abortTask()` also signals and joins an active run before starting the abort
invocation. `Conversation.abort()` withdraws queued input submissions, marks
non-background tasks in the conversation and owned subtree, signals them, and
resolves only after that subtree is ordinarily idle. Passive writes and
background tasks survive. Conversation idle means no non-background live task
in that conversation or its ownership subtree; Harness idle applies the same
rule Session-wide. Pending dependency- or deadline-blocked work is still live
and therefore not idle. Cancelling an idle wait aborts only that waiter.
`markTask()` commits `abortRequested` and the durable foreground-subtree
cascade; it neither signals nor joins active invocations. The scheduler notices
the marks on its next drain. `abortTask()` also signals and joins the active run
before starting the abort invocation. `Conversation.abort()` withdraws queued
input submissions, marks non-background tasks selected by ordinary ownership
traversal, signals them, and resolves only after that scope is ordinarily idle.
Passive writes and background subtrees survive. Conversation idle means no live
non-background task selected from that conversation. Harness idle applies the
same traversal from every ownerless conversation root. Pending dependency- or
deadline-blocked work is still live and therefore not idle. Cancelling an idle
wait aborts only that waiter.
`onConversation()` synchronously visits the currently loaded committed
conversations in ascending ID order, then reports each later creation after its
@ -736,21 +769,21 @@ A persisted document instance has one `DocumentRecord`:
```ts
type DocumentRecord = {
readonly id: Id; // unique incarnation
readonly id: DocumentId; // unique incarnation
readonly kind: string; // stable definition kind
readonly key?: string; // families only
readonly createdAt: Seq; // stamped by the committing storage
readonly retiredAt?: Seq;
} & (
| { readonly scope: { readonly kind: "session" } }
| ({ readonly scope: { readonly kind: "conversation"; readonly conversationId: Id } } & (
| ({ readonly scope: { readonly kind: "conversation"; readonly conversationId: ConversationId } } & (
| { readonly history: "latest"; readonly fork: "current" | "initial" }
| {
readonly history: "rewindable";
readonly fork: "asOf" | "current" | "initial";
}
))
| { readonly scope: { readonly kind: "task"; readonly taskId: Id } }
| { readonly scope: { readonly kind: "task"; readonly taskId: TaskId } }
);
type DocumentCreate = DocumentRecord extends infer Record
@ -783,55 +816,72 @@ interface Session extends DocumentObserver {
close(context: Context): Promise<void>;
snapshot<T extends JsonObject>(token: SessionDocToken<T>, context: Context): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: Id, context: Context): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject>(token: TaskDocToken<T>, taskId: Id, context: Context): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: ConversationId, context: Context): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject>(token: TaskDocToken<T>, taskId: TaskId, context: Context): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject, I extends JsonValue>(token: SessionDocFamilyToken<T, I>, key: string, context: Context): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject, I extends JsonValue>(token: ConversationDocFamilyToken<T, I>, conversationId: Id, key: string, context: Context): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject, I extends JsonValue>(token: TaskDocFamilyToken<T, I>, taskId: Id, key: string, context: Context): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject, I extends JsonValue>(token: ConversationDocFamilyToken<T, I>, conversationId: ConversationId, key: string, context: Context): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject, I extends JsonValue>(token: TaskDocFamilyToken<T, I>, taskId: TaskId, key: string, context: Context): Promise<Readonly<T> | undefined>;
snapshotAsOf<T extends JsonObject>(token: RewindableConversationDocToken<T>, conversationId: Id, at: Id, context: Context): Promise<Readonly<T> | undefined>;
snapshotAsOf<T extends JsonObject, I extends JsonValue>(token: RewindableConversationDocFamilyToken<T, I>, conversationId: Id, key: string, at: Id, context: Context): Promise<Readonly<T> | undefined>;
snapshotAsOf<T extends JsonObject>(token: RewindableConversationDocToken<T>, conversationId: ConversationId, at: EntryId, context: Context): Promise<Readonly<T> | undefined>;
snapshotAsOf<T extends JsonObject, I extends JsonValue>(token: RewindableConversationDocFamilyToken<T, I>, conversationId: ConversationId, key: string, at: EntryId, context: Context): Promise<Readonly<T> | undefined>;
documentSource<T extends JsonObject>(token: SessionDocToken<T>, context: Context): Promise<DocumentSource<T> | undefined>;
documentSource<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: Id, context: Context): Promise<DocumentSource<T> | undefined>;
documentSource<T extends JsonObject>(token: TaskDocToken<T>, taskId: Id, context: Context): Promise<DocumentSource<T> | undefined>;
documentSource<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: ConversationId, context: Context): Promise<DocumentSource<T> | undefined>;
documentSource<T extends JsonObject>(token: TaskDocToken<T>, taskId: TaskId, context: Context): Promise<DocumentSource<T> | undefined>;
documentSource<T extends JsonObject, I extends JsonValue>(token: SessionDocFamilyToken<T, I>, key: string, context: Context): Promise<DocumentSource<T> | undefined>;
documentSource<T extends JsonObject, I extends JsonValue>(token: ConversationDocFamilyToken<T, I>, conversationId: Id, key: string, context: Context): Promise<DocumentSource<T> | undefined>;
documentSource<T extends JsonObject, I extends JsonValue>(token: TaskDocFamilyToken<T, I>, taskId: Id, key: string, context: Context): Promise<DocumentSource<T> | undefined>;
documentSource<T extends JsonObject, I extends JsonValue>(token: ConversationDocFamilyToken<T, I>, conversationId: ConversationId, key: string, context: Context): Promise<DocumentSource<T> | undefined>;
documentSource<T extends JsonObject, I extends JsonValue>(token: TaskDocFamilyToken<T, I>, taskId: TaskId, key: string, context: Context): Promise<DocumentSource<T> | undefined>;
}
interface Tx {
conversation(id: Id): Promise<ConversationRecord | undefined>;
entry(id: Id): Promise<EntryRecord | undefined>;
task(id: Id): Promise<TaskRecord<JsonValue, JsonValue, JsonValue> | undefined>;
scanConversations(limit: number, cursor?: Cursor): Promise<Page<ConversationRecord, Cursor>>;
conversation(id: ConversationId): Promise<ConversationRecord | undefined>;
entry(id: EntryId): Promise<EntryRecord | undefined>;
task(id: TaskId): Promise<TaskRecord<JsonValue, JsonValue, JsonValue> | undefined>;
scanConversations(query: ConversationQuery, limit: number, cursor?: Cursor): Promise<Page<ConversationRecord, Cursor>>;
scanEntries(query: EntryQuery, limit: number, cursor?: Cursor): Promise<Page<EntryRecord, Cursor>>;
scanTasks(query: TaskQuery, limit: number, cursor?: Cursor): Promise<Page<TaskRecord<JsonValue, JsonValue, JsonValue>, Cursor>>;
createConversation(value: Omit<ConversationRecord, "id">): Promise<ConversationRecord>;
appendEntry(conversationId: Id, value: EntryDraft): Promise<EntryRecord>;
createConversation(options: { readonly ownership: ConversationOwnership }): Promise<ConversationRecord>;
forkConversation(
parentConversationId: ConversationId,
at: EntryId,
options: { readonly ownership: ConversationOwnership },
): Promise<ConversationRecord>;
appendEntry(conversationId: ConversationId, value: EntryDraft): Promise<EntryRecord>;
createTask<I, S extends { phase: string }, R, H extends object>(
task: Task<I, S, R, H>, input: I, options?: TaskOptions,
): Promise<TaskRef<R>>;
): Promise<TaskId<R>>;
setTask(value: TaskRecord<JsonValue, JsonValue, JsonValue>): void;
doc<T extends JsonObject>(token: SessionDocToken<T>): Promise<Draft<T>>;
doc<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: Id): Promise<Draft<T>>;
doc<T extends JsonObject>(token: TaskDocToken<T>, taskId: Id): Promise<Draft<T>>;
doc<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: ConversationId): Promise<Draft<T>>;
doc<T extends JsonObject>(token: TaskDocToken<T>, taskId: TaskId): Promise<Draft<T>>;
doc<T extends JsonObject, I extends JsonValue>(token: SessionDocFamilyToken<T, I>, key: string, seed: I): Promise<Draft<T>>;
doc<T extends JsonObject, I extends JsonValue>(token: ConversationDocFamilyToken<T, I>, conversationId: Id, key: string, seed: I): Promise<Draft<T>>;
doc<T extends JsonObject, I extends JsonValue>(token: TaskDocFamilyToken<T, I>, taskId: Id, key: string, seed: I): Promise<Draft<T>>;
doc<T extends JsonObject, I extends JsonValue>(token: ConversationDocFamilyToken<T, I>, conversationId: ConversationId, key: string, seed: I): Promise<Draft<T>>;
doc<T extends JsonObject, I extends JsonValue>(token: TaskDocFamilyToken<T, I>, taskId: TaskId, key: string, seed: I): Promise<Draft<T>>;
retireDoc<T extends JsonObject>(token: SessionDocToken<T>): Promise<void>;
retireDoc<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: Id): Promise<void>;
retireDoc<T extends JsonObject>(token: TaskDocToken<T>, taskId: Id): Promise<void>;
retireDoc<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: ConversationId): Promise<void>;
retireDoc<T extends JsonObject>(token: TaskDocToken<T>, taskId: TaskId): Promise<void>;
retireDoc<T extends JsonObject, I extends JsonValue>(token: SessionDocFamilyToken<T, I>, key: string): Promise<void>;
retireDoc<T extends JsonObject, I extends JsonValue>(token: ConversationDocFamilyToken<T, I>, conversationId: Id, key: string): Promise<void>;
retireDoc<T extends JsonObject, I extends JsonValue>(token: TaskDocFamilyToken<T, I>, taskId: Id, key: string): Promise<void>;
retireDoc<T extends JsonObject, I extends JsonValue>(token: ConversationDocFamilyToken<T, I>, conversationId: ConversationId, key: string): Promise<void>;
retireDoc<T extends JsonObject, I extends JsonValue>(token: TaskDocFamilyToken<T, I>, taskId: TaskId, key: string): Promise<void>;
}
```
ID-creating transaction methods are asynchronous because remote storage may allocate globally unique IDs durably. Only public typed `tx.doc()` is get-or-create. Internal fork copying may create
ID-creating transaction methods are asynchronous because remote storage may
allocate globally unique numeric IDs durably. Conversation creation always
requires explicit ownership; no transaction wrapper injects the executing task.
For task ownership, the caller supplies only a typed task ID. The Session derives
the persisted owner conversation from the task's final candidate record.
An owner task may be committed or staged earlier in the same transaction. Before
Storage admission, the Session rejects a missing, terminal, or abort-marked owner,
including one made terminal or abort-marked later in that transaction. Existing
owner edges remain valid when their owners terminalize afterward. Conversation
creation returns an inert record, never an operational handle.
`forkConversation()` additionally validates one visible entry and applies section
3.7. Only public typed `tx.doc()` is get-or-create. Internal fork copying may create
new incarnations directly from stored values without a definition. `tx.doc()`
receives the definition token that supplies
its static type, initializer, migration, and checkpoint policy. Scope-preserving
@ -1084,7 +1134,7 @@ Each conversation document follows the history/fork policy persisted in its
| conversation setting | child value |
|---|---|
| `fork: "asOf"` | parent value at `E`'s commit |
| `fork: "current"` | parent value when the fork commit runs |
| `fork: "current"` | committed parent value selected when the fork commit runs |
| `fork: "initial"` | no copied instance; initializer on first child access |
`current` and `asOf` copy logically present conversation singleton and family
@ -1094,6 +1144,32 @@ values become independent child instances with new IDs and initial bases.
supplied definition. Task documents and tasks are never copied. Session documents
remain shared and are not rewindable.
Fork copying reads committed stored values rather than typed tracker caches. A
transaction that creates a fork therefore rejects if it also writes one of the
parent's `fork: "current"` documents; commit the parent change first so the fork
has one unambiguous stored source revision.
Forks stage backend-side `document.copy` commands carrying the child create
record and an exact source incarnation/point. Storage materializes each source
and persists its stored value/version as the child's independent initial base;
remote Storage performs this server-side. Every copy reads committed pre-batch
source state independent of write-array order. A selected source may not be
created, changed, or retired in the same batch. Copying remains atomic with the
conversation, overrides, registry writes, and other mutations.
An unaccessed copy retains only its descriptor in Session memory. Typed access
inside the creating transaction lazily reads the detached source, migrates when
required, and replaces the copy with one ordinary child create containing the
final prepared value. Definition-free copies publish explicit `document.copy`
metadata rather than a value. That metadata announces Storage-backed initial
state and is never interpreted as a document value. Document sources, watches,
and mounted views hydrate by capturing their baseline and subscription
atomically on the Session line: a later commit already present becomes the
baseline, while one committed after registration is delivered. Publications
and watches are convergence mechanisms, not audit streams. A mounted aggregate
must acquire all of its document baselines and commit subscription in one
Session-line operation so it never exposes a mixture from one commit.
## 4. Transactions and storage ownership
A Session commit callback may be asynchronous. It owns the Session mutation
@ -1152,12 +1228,12 @@ type TaskState<S, R> =
| { readonly status: "terminal"; readonly outcome: TaskOutcome<R> };
type TaskRecord<I, S, R> = {
readonly id: Id;
readonly conversationId: Id;
readonly id: TaskId<R>;
readonly conversationId: ConversationId;
readonly kind: string;
readonly version: number;
readonly input: I;
readonly after: readonly Id[];
readonly after: readonly TaskId[];
readonly background: boolean;
readonly abortRequested: boolean;
} & (
@ -1186,8 +1262,8 @@ type PhaseHandler<I, P, S, R, H extends object> = (
) => Promise<void>;
interface TaskRuntime<I, S, R, H extends object> extends DocumentObserver {
readonly taskId: Id;
readonly conversationId: Id;
readonly taskId: TaskId<R>;
readonly conversationId: ConversationId;
readonly signal: AbortSignal;
readonly hooks: HookRunner<H>;
@ -1198,6 +1274,7 @@ interface TaskRuntime<I, S, R, H extends object> extends DocumentObserver {
memo<T extends JsonValue>(name: string, context: Context): Promise<T | undefined>;
memo<T extends JsonValue>(name: string, candidate: T, context: Context): Promise<T>;
conversation(id: ConversationId, context: Context): Promise<ConversationHandle | undefined>;
sleep(until: number, context: Context): Promise<void>;
}
@ -1220,11 +1297,9 @@ interface Task<I, S extends { phase: string }, R, H extends object> {
readonly definition: TaskDefinition<I, S, R, H>;
}
declare const taskResultType: unique symbol;
type TaskRef<R> = { readonly id: Id; readonly [taskResultType]?: R };
type TaskOptions = {
readonly conversationId?: Id;
readonly after?: readonly Id[];
readonly conversationId?: ConversationId;
readonly after?: readonly TaskId[];
readonly background?: boolean;
};
@ -1233,6 +1308,13 @@ function defineTask<I, S extends { phase: string }, R, H extends object = {}>(
): Task<I, S, R, H>;
```
`TaskId<R>` is the numeric task ID itself with an erased result-type brand. It
replaces a separate task-reference wrapper: `createTask()` returns `TaskId<R>`,
typed waits infer `R` from that ID, and result-agnostic operations accept
`TaskId<unknown>`. A task's `conversationId` is immutable after creation; a
replacement that moves an existing task to another conversation rejects before
Storage admission so persisted conversation-owner edges cannot become stale.
The phase map is exhaustive and phase-narrowed. A handler may perform several
commits around one effect, but each durable checkpoint is a full replacement.
`TaskRuntime.commit()` rereads and gates the current durable task on the Session
@ -1329,10 +1411,35 @@ the durable mark on reopen. Cancelling one caller's `Context` only cancels
that call or wait; it does not durably abort shared work unless the invoked API
commits an abort mark.
A task may create owned conversations. Abort and idle operations traverse the
conversation ownership tree. History parents are irrelevant to this traversal.
Background tasks do not block ordinary idle waits and survive ordinary
conversation abort unless explicitly included.
A task may create owned conversations. Conversations are durable scopes; tasks
are the units of live work counted by idle and marked by abort. History parents
are irrelevant to ownership traversal.
Ordinary traversal starts at an explicitly addressed conversation, visits its
tasks, and follows conversations owned by each non-background task. It follows
owner edges after the owner becomes terminal, but a background owner is a
boundary: ordinary traversal skips that task and its complete owned subtree.
Direct conversation operations start inside that conversation regardless of its
owner. Directly aborting a live background task includes that task and follows
its ordinary owned subtree; nested background owners remain boundaries. Full
teardown crosses every boundary, marks every live task, and must seal new
admission while it gathers the complete indexed ownership subtree.
`Conversation.abort()` withdraws queued inputs and marks live non-background
tasks selected by ordinary traversal. `Conversation.waitForIdle()` waits until
that traversal contains no live non-background task. Harness idle performs the
same traversal from every ownerless conversation root rather than globally
counting tasks, so ordinary work below a background owner does not block it.
Explicit `waitForTask()` waits for its referenced task regardless of the task's
background flag.
An abort mark atomically and idempotently cascades to foreground-owned work,
including conversations, tasks, and submissions staged in the same transaction.
Terminal outcomes `failed`, `faulted`, `orphaned`, and `aborted` record the same
durable cancellation intent; `completed` does not. Conversation records and
owner edges are never retired with the task. Active invocations are signalled
after commit, and a terminal receipt guarantees durable cancellation intent,
not descendant quiescence.
Initial task definitions are registered before open performs live-task migration
and orphan reconciliation. Dynamic registration begins only after that pass.
@ -1351,8 +1458,8 @@ itself is an ordered conversation document containing tagged items:
```ts
type InboxItem =
| { readonly id: Id; readonly mode: "steer" | "followUp"; readonly message: Message }
| { readonly id: Id; readonly mode: "write"; readonly entry: EntryDraft };
| { readonly id: SubmissionId; readonly mode: "steer" | "followUp"; readonly message: Message }
| { readonly id: SubmissionId; readonly mode: "write"; readonly entry: EntryDraft };
```
A built-in turn-control document has an optional `active` value naming the task
@ -1447,16 +1554,16 @@ type ToolExecutionResult = {
readonly control?: ToolControl;
};
interface OwnedConversation {
readonly id: Id;
interface ConversationHandle {
readonly id: ConversationId;
submit(submission: InputSubmissionDraft, context: Context): Promise<Submission>;
abort(context: Context): Promise<void>;
waitForIdle(context: Context): Promise<void>;
}
interface ToolExecutionApi extends DocumentObserver {
readonly taskId: Id;
readonly conversationId: Id;
readonly taskId: TaskId;
readonly conversationId: ConversationId;
readonly callId: string;
stream(chunk: string | Uint8Array): void;
commit<T>(change: (tx: Tx) => T | Promise<T>, context: Context): Promise<T>;
@ -1468,13 +1575,10 @@ interface ToolExecutionApi extends DocumentObserver {
input: I,
options: Omit<TaskOptions, "conversationId">,
context: Context,
): Promise<TaskRef<R>>;
getTask<R>(ref: TaskRef<R>, context: Context): Promise<TaskRecord<JsonValue, JsonValue, R> | undefined>;
waitForTask<R>(ref: TaskRef<R>, context: Context): Promise<SettledTask<R>>;
createConversation(
spec: Omit<ConversationSpec, "parent"> & { readonly inherit?: boolean },
context: Context,
): Promise<OwnedConversation>;
): Promise<TaskId<R>>;
getTask<R>(id: TaskId<R>, context: Context): Promise<TaskRecord<JsonValue, JsonValue, R> | undefined>;
waitForTask<R>(id: TaskId<R>, context: Context): Promise<SettledTask<R>>;
conversation(id: ConversationId, context: Context): Promise<ConversationHandle | undefined>;
}
type ToolRegistration = Tool & {
@ -1523,35 +1627,82 @@ validated before and after `beforeTool` hooks.
After hooks and validation, the tool task durably records the final call and
resolved replay policy before execution. Recovery does not rerun `beforeTool`
and does not let a changed registry declaration alter that stored policy.
and passes the same stored arguments to `execute()`. A replay-safe tool may
reconstruct a submission from those arguments when that transformation is pure.
Random values, timestamps, mutable document/configuration reads, or other derived
inputs that must remain stable are first captured in a durable memo, checkpoint,
or task input. A background supervisor receives the final submission draft in
its own durable input so it can finish independently. Recovery does not let a
changed registry declaration alter the stored replay policy.
An owned conversation accepts only input submissions; tools use ordinary
transaction writes for passive entries.
A tool-acquired conversation handle accepts only input submissions; tools use
ordinary transaction writes for passive entries.
A tool executes in a durable task. It may:
- write bounded progress/output to a presentation or task-scoped document;
- commit memos;
- create and wait for tasks;
- create owned conversations;
- atomically create or fork explicitly owned conversations through `commit()`;
- observe documents for which it has a token/reference;
- mutate authorized documents through `commit()`;
- return bounded model content and separate diagnostic details.
Tool operations use the invoking task's admission and
invocation-lifetime gates. Task creation defaults to that task's conversation.
Trusted document access follows sections 3.3 and 9.2; there is no additional
document subtree authorization layer. `createConversation()` records the
invoking task as owner and returns an invocation-bound handle. `inherit` defaults
to `false`. When true, creation atomically selects the invoking conversation's
newest committed visible entry as `parent.at`; if no entry exists, it creates no
history parent. Conversation documents then apply section 3.7 at that entry, and
explicit model/section seeds override their inherited built-in values. With no
parent, documents initialize normally. That handle's
operations reject after the invocation ends. A `Submission` returned by its
`submit()` is invocation-bound in the same way; the admitted submission remains
durable after those methods reject. Invocation-owned document watches stop when
the invocation ends.
Tool operations use the invoking task's admission and invocation-lifetime gates.
Task creation defaults to that task's conversation. Trusted document and
conversation access follows sections 3.3 and 9.2; there is no additional subtree
authorization layer. Possession of a Session-global typed ID is sufficient in
trusted code.
Tools have no conversation-creation convenience method. They create or fork a
conversation inside `commit()`, state its ownership explicitly, atomically stage
any related document/task writes, and receive only the inert record. After that
commit settles, `conversation(id)` acquires an invocation-bound operational
handle for submission, explicit abort, and idle waits without changing
ownership. A commit callback must not call `conversation()` or use a previously
acquired handle: nested Session operations and external effects are forbidden
while the mutation line is held. A handle's operations reject after the
invocation ends. A `Submission` returned by its `submit()` is invocation-bound in
the same way; the admitted submission remains durable after those methods reject.
Invocation-owned document watches stop when the invocation ends.
A foreground subagent conversation is explicitly owned by its tool task. One
transaction creates or forks the child and records its durable registry mapping;
after settlement the tool reacquires the child, submits with the registered
request ID, and waits for that submission's result. Aborting or abnormally
terminalizing the tool task cascades through the owned scope.
A background subagent is provisioned in one transaction. After deduplicating by
its durable registry key, the tool stages a background supervisor task `B`, a
child conversation `C` explicitly owned by `B`, and the registry mapping from the
semantic name to `C` plus its stable request ID. `B` may be staged earlier in the
same transaction and used immediately as `C`'s owner. Its durable input contains
the exact submission draft and the registry location/key; it need not contain
`C`'s not-yet-created ID. Once scheduled, `B` resolves `C` from the mapping,
verifies the immutable owner edge, and performs normal `Conversation.submit()`.
A crash before admission makes `B` retry; a crash after admission returns the
existing request-ID-deduplicated `Submission`. The initiating tool may race the
same submit for lower latency and wait only for the durable admission receipt.
The supervisor may complete after setup: its terminal record retains
`background`, so ancestor ordinary abort and idle traversal continue to stop at
its owned scope.
Applications may maintain a conversation document mapping semantic subagent
names to durable conversation IDs and application-minted submission request IDs.
Such a live registry uses `fork: "initial"` so children do not inherit the
parent's agent list and its update is not a selected fork source. Conversation
creation/forking, supervisor creation when applicable, and the registry mapping
commit atomically. The supervisor input, tool's durable final arguments, or a
durable checkpoint must retain enough information to reconstruct the exact
submission draft; the registry itself need not duplicate that payload. Initial
or later input then uses the registered stable request ID with the full
`Conversation.submit()` state machine. A crash before admission leaves a durable
request to submit; a crash after admission retries the same request and receives
the existing `Submission`. A read-only lookup by conversation/request ID can
report absence or return the durable queued/placed/done/unanswered receipt.
Submission admission therefore needs no private transaction shortcut and does
not move onto `Tx`. Kernel ownership indexes independently drive abort and idle
traversal.
A tool result may request `addTools`, `terminate`, or `handoff`. Post-tools
applies added tool names to configured loadout, uses a final boundary for
@ -1782,11 +1933,11 @@ type DocumentWatch<T extends JsonObject> = WatchHandle<Readonly<T> | null>;
interface DocumentObserver {
watchDoc<T extends JsonObject>(token: SessionDocToken<T>, context: Context): Promise<DocumentWatch<T> | undefined>;
watchDoc<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: Id, context: Context): Promise<DocumentWatch<T> | undefined>;
watchDoc<T extends JsonObject>(token: TaskDocToken<T>, taskId: Id, context: Context): Promise<DocumentWatch<T> | undefined>;
watchDoc<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: ConversationId, context: Context): Promise<DocumentWatch<T> | undefined>;
watchDoc<T extends JsonObject>(token: TaskDocToken<T>, taskId: TaskId, context: Context): Promise<DocumentWatch<T> | undefined>;
watchDoc<T extends JsonObject, I extends JsonValue>(token: SessionDocFamilyToken<T, I>, key: string, context: Context): Promise<DocumentWatch<T> | undefined>;
watchDoc<T extends JsonObject, I extends JsonValue>(token: ConversationDocFamilyToken<T, I>, conversationId: Id, key: string, context: Context): Promise<DocumentWatch<T> | undefined>;
watchDoc<T extends JsonObject, I extends JsonValue>(token: TaskDocFamilyToken<T, I>, taskId: Id, key: string, context: Context): Promise<DocumentWatch<T> | undefined>;
watchDoc<T extends JsonObject, I extends JsonValue>(token: ConversationDocFamilyToken<T, I>, conversationId: ConversationId, key: string, context: Context): Promise<DocumentWatch<T> | undefined>;
watchDoc<T extends JsonObject, I extends JsonValue>(token: TaskDocFamilyToken<T, I>, taskId: TaskId, key: string, context: Context): Promise<DocumentWatch<T> | undefined>;
}
```
@ -1973,14 +2124,19 @@ type Page<T, C> = {
type Cursor = Readonly<Record<string, JsonValue>>;
type ConversationQuery = {
readonly ownerConversationId?: ConversationId;
readonly ownerTaskId?: TaskId;
};
type EntryQuery = {
readonly conversationId: Id;
readonly minEntryId?: Id; // inclusive
readonly maxEntryId?: Id; // inclusive
readonly conversationId: ConversationId;
readonly minEntryId?: EntryId; // inclusive
readonly maxEntryId?: EntryId; // inclusive
};
type TaskQuery = {
readonly conversationId?: Id;
readonly conversationId?: ConversationId;
readonly kind?: string;
readonly status?: "pending" | "running" | "terminal";
readonly abortRequested?: boolean;
@ -2021,12 +2177,17 @@ type StorageWrite =
readonly record: DocumentCreate;
readonly content: Extract<DocumentContent, { kind: "base" }>;
}
| {
readonly type: "document.copy";
readonly record: DocumentCreate;
readonly source: { readonly id: DocumentId; readonly at: DocumentPoint };
}
| {
readonly type: "document.change";
readonly id: Id;
readonly id: DocumentId;
readonly content: DocumentContent;
}
| { readonly type: "document.retire"; readonly id: Id };
| { readonly type: "document.retire"; readonly id: DocumentId };
/**
* Trusts the owning Session to supply semantically valid records, references,
@ -2037,30 +2198,44 @@ type StorageWrite =
*/
interface Storage {
commit(writes: readonly StorageWrite[], context: Context): Promise<Seq>;
mintId(): Promise<Id>;
/** Allocate from the one global numeric namespace; the generic brand is compile-time only. */
mintId<I extends Id<string>>(): Promise<I>;
conversation(id: Id, context: Context): Promise<ConversationRecord | undefined>;
scanConversations(limit: number, cursor: Cursor | undefined, context: Context): Promise<Page<ConversationRecord, Cursor>>;
conversation(id: ConversationId, context: Context): Promise<ConversationRecord | undefined>;
scanConversations(query: ConversationQuery, limit: number, cursor: Cursor | undefined, context: Context): Promise<Page<ConversationRecord, Cursor>>;
entry(id: Id, context: Context): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
entry(conversationId: Id, id: Id, context: Context): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
findLatestHeadMarker(conversationId: Id, atOrBeforeEntryId: Id | undefined, context: Context): Promise<(EntryRecord & { readonly head: Id }) | undefined>;
entry(id: EntryId, context: Context): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
entry(conversationId: ConversationId, id: EntryId, context: Context): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
findLatestHeadMarker(conversationId: ConversationId, atOrBeforeEntryId: EntryId | undefined, context: Context): Promise<(EntryRecord & { readonly head: EntryId }) | undefined>;
scanEntries(query: EntryQuery, limit: number, cursor: Cursor | undefined, context: Context): Promise<Page<EntryRecord, Cursor>>;
task(id: Id, context: Context): Promise<TaskRecord<JsonValue, JsonValue, JsonValue> | undefined>;
task(id: TaskId, context: Context): Promise<TaskRecord<JsonValue, JsonValue, JsonValue> | undefined>;
scanTasks(query: TaskQuery, limit: number, cursor: Cursor | undefined, context: Context): Promise<Page<TaskRecord<JsonValue, JsonValue, JsonValue>, Cursor>>;
submission(id: Id, context: Context): Promise<SubmissionRecord | undefined>;
submissionByRequest(conversationId: Id, requestId: string, context: Context): Promise<SubmissionRecord | undefined>;
submission(id: SubmissionId, context: Context): Promise<SubmissionRecord | undefined>;
submissionByRequest(conversationId: ConversationId, requestId: string, context: Context): Promise<SubmissionRecord | undefined>;
findDocument(address: DocumentAddress, at: DocumentPoint, context: Context): Promise<DocumentRecord | undefined>;
document(id: Id, at: DocumentPoint, context: Context): Promise<StoredDocument | undefined>;
document(id: DocumentId, at: DocumentPoint, context: Context): Promise<StoredDocument | undefined>;
scanDocuments(query: DocumentQuery, limit: number, cursor: Cursor | undefined, context: Context): Promise<Page<DocumentRecord, Cursor>>;
close(context: Context): Promise<void>;
}
```
`StorageRejected` means a batch was rejected before any durable effect and is
guaranteed not to have committed. Session rolls such a batch back normally;
unknown failures after Storage admission remain fatal because their commit state
is uncertain. Backends use `StorageRejected` for deterministic `document.copy`
source, replay, and consistency failures only when rollback is guaranteed.
A `document.copy` reads committed pre-batch source state independent of command
order. The source must be an alive conversation document at the selected point,
and kind/key/history/fork must match the child create record. Storage persists
one independent complete child base at the source's stored version. A batch may
not create, change, or retire a selected source. Later source changes,
reclamation, retirement, or backend reopen cannot affect the child.
Cursors are backend-owned JSON objects. Callers only round-trip them to the same
scan on the same storage; cross-storage or cross-query use is unsupported. The
Session owns the mutation line, so storage implementations do not add a second
@ -2073,6 +2248,10 @@ inclusive ID range in newest-first order while applying every conversation
ancestry cap. With no bounds it pages complete visible history. To read context
through entry `E`, find the marker at or before `E`, then scan from
`marker?.head` through `E`. For current context the upper bound is omitted.
Conversation owner filters are indexed and conjunctive. They support ownership
traversal without an all-conversation scan; application-maintained registries
are not a substitute for these kernel indexes.
`entry(id)` combines exact global lookup with the commit sequence required by
historical document reads. `entry(conversationId, id)` returns that pair only
when the entry is visible through the requested conversation's ancestry. `limit`

View file

@ -1,15 +1,18 @@
import { copyJson, type JsonValue } from "@earendil-works/chord";
import type { Op } from "@earendil-works/chord/delta";
import { idFromNumber } from "./ids.ts";
import type {
CommonDocDefinition,
ConversationDocFamilyToken,
ConversationDocToken,
ConversationId,
DocDefinition,
DocFamilyDefinition,
DocFamilyToken,
DocToken,
DocumentAddress,
DocumentCreate,
DocumentId,
DocumentRecord,
DocumentSemantics,
Id,
@ -23,6 +26,7 @@ import type {
StoredDocument,
TaskDocFamilyToken,
TaskDocToken,
TaskId,
} from "./types.ts";
type FamilyInput<T extends JsonObject, I extends JsonValue> = Omit<CommonDocDefinition<T>, "initial"> & {
@ -109,10 +113,10 @@ export function resolveAddress(definition: AnyDocDefinition, args: readonly unkn
scope = { kind: "session" };
break;
case "conversation":
scope = { kind: "conversation", conversationId: ownerId(args[index++], definition) };
scope = { kind: "conversation", conversationId: ownerId<ConversationId>(args[index++], definition) };
break;
case "task":
scope = { kind: "task", taskId: ownerId(args[index++], definition) };
scope = { kind: "task", taskId: ownerId<TaskId>(args[index++], definition) };
break;
}
const key = definition.family === true ? (args[index++] as string) : undefined;
@ -121,11 +125,11 @@ export function resolveAddress(definition: AnyDocDefinition, args: readonly unkn
return { address, id: addressId(address), nextArgument: index };
}
function ownerId(value: unknown, definition: AnyDocDefinition): Id {
if (!Number.isSafeInteger(value as number)) {
function ownerId<I extends Id<string>>(value: unknown, definition: AnyDocDefinition): I {
if (typeof value !== "number" || !Number.isSafeInteger(value)) {
throw new TypeError(`Document ${definition.kind} requires a ${definition.scope} ID`);
}
return value as Id;
return idFromNumber<I>(value);
}
/** Stable string identity of one logical address. */
@ -140,7 +144,7 @@ export function addressId(address: DocumentAddress): string {
}
/** Build the storage create record for a new incarnation at an address. */
export function documentCreate(definition: AnyDocDefinition, address: DocumentAddress, id: Id): DocumentCreate {
export function documentCreate(definition: AnyDocDefinition, address: DocumentAddress, id: DocumentId): DocumentCreate {
const key = address.key === undefined ? {} : { key: address.key };
switch (address.scope.kind) {
case "session":
@ -160,7 +164,7 @@ export function documentCreate(definition: AnyDocDefinition, address: DocumentAd
}
/** Reject typed access whose token disagrees with the persisted scope, history, or fork semantics. */
export function checkRecordScope(definition: AnyDocDefinition, record: DocumentRecord): void {
export function checkRecordScope(definition: AnyDocDefinition, record: DocumentCreate | DocumentRecord): void {
if (
record.scope.kind !== definition.scope ||
(record.scope.kind === "conversation" &&
@ -171,7 +175,11 @@ export function checkRecordScope(definition: AnyDocDefinition, record: DocumentR
}
/** Reject typed access to a stored version the supplied definition cannot use. */
export function checkRecordVersion(definition: AnyDocDefinition, record: DocumentRecord, version: number): void {
export function checkRecordVersion(
definition: AnyDocDefinition,
record: DocumentCreate | DocumentRecord,
version: number,
): void {
if (version > definition.version) {
throw new Error(`Document ${record.id} (${record.kind}) has newer version ${version} than ${definition.version}`);
}
@ -182,8 +190,18 @@ export function checkRecordVersion(definition: AnyDocDefinition, record: Documen
/** Validate and materialize a detached stored value for typed access. */
export function materializeDocument(definition: AnyDocDefinition, stored: StoredDocument): JsonObject {
checkRecordScope(definition, stored.record);
checkRecordVersion(definition, stored.record, stored.version);
if (stored.version === definition.version) return stored.value;
return copyJson(definition.migrate!(stored.value, stored.version)) as JsonObject;
return materializeDocumentValue(definition, stored.record, stored.version, stored.value);
}
/** Validate and materialize one detached value before its first persisted incarnation. */
export function materializeDocumentValue(
definition: AnyDocDefinition,
record: DocumentCreate | DocumentRecord,
version: number,
value: JsonObject,
): JsonObject {
checkRecordScope(definition, record);
checkRecordVersion(definition, record, version);
if (version === definition.version) return value;
return copyJson(definition.migrate!(value, version)) as JsonObject;
}

View file

@ -0,0 +1,15 @@
/** A transaction read a table after its first table write. Read every required row before writing. */
export class ReadAfterWrite extends Error {
constructor(method: string) {
super(`Tx.${method}() cannot read tables after the first table write`);
this.name = "ReadAfterWrite";
}
}
/** Storage rejected a batch before any durable effect; the owning Session may continue safely. */
export class StorageRejected extends Error {
constructor(message: string, options?: ErrorOptions) {
super(message, options);
this.name = "StorageRejected";
}
}

View file

@ -0,0 +1,11 @@
import type { Id, Seq } from "./types.ts";
/** Apply an erased ID brand at a trusted numeric allocation or decoding boundary. */
export function idFromNumber<I extends Id<string>>(value: number): I {
return value as I;
}
/** Apply the erased commit-sequence brand at a trusted storage boundary. */
export function seqFromNumber(value: number): Seq {
return value as Seq;
}

View file

@ -1,12 +1,15 @@
export { defineDoc, defineDocFamily } from "./documents.ts";
export { ReadAfterWrite, StorageRejected } from "./errors.ts";
export { createSession } from "./session/session.ts";
export { ReadAfterWrite } from "./session/transaction.ts";
export { MemoryStorage } from "./storage/memory.ts";
export type {
CommonDocDefinition,
ContextEdit,
ConversationDocFamilyToken,
ConversationDocToken,
ConversationId,
ConversationOwnership,
ConversationQuery,
ConversationRecord,
Cursor,
DocDefinition,
@ -15,12 +18,15 @@ export type {
DocToken,
DocumentAddress,
DocumentContent,
DocumentCopySource,
DocumentCreate,
DocumentId,
DocumentPoint,
DocumentQuery,
DocumentRecord,
DocumentSemantics,
EntryDraft,
EntryId,
EntryQuery,
EntryRecord,
Id,
@ -38,17 +44,18 @@ export type {
StorageWrite,
StoredDocument,
SubmissionCreate,
SubmissionId,
SubmissionRecord,
Task,
TaskDefinition,
TaskDocFamilyToken,
TaskDocToken,
TaskId,
TaskOptions,
TaskOutcome,
TaskOutcomeError,
TaskQuery,
TaskRecord,
TaskRef,
TaskState,
Tx,
} from "./types.ts";

View file

@ -0,0 +1,97 @@
import type { Context } from "@earendil-works/chord";
import { addressId } from "../documents.ts";
import type {
ConversationId,
Cursor,
DocumentCopySource,
DocumentCreate,
DocumentId,
DocumentPoint,
DocumentRecord,
EntryId,
Storage,
} from "../types.ts";
const SCAN_PAGE_SIZE = 256;
type ForkPolicy = "asOf" | "current";
/** One definition-free document copy to create with a forked conversation. */
export type ForkDocumentCopy = {
readonly record: DocumentCreate;
readonly source: DocumentCopySource;
};
/** Select every persisted conversation document copied by one fork. */
export async function prepareForkDocumentCopies(
storage: Storage,
parentConversationId: ConversationId,
at: EntryId,
childConversationId: ConversationId,
context: Context,
): Promise<readonly ForkDocumentCopy[]> {
const entry = await storage.entry(parentConversationId, at, context);
if (entry === undefined) throw new Error(`Entry ${at} is not visible from conversation ${parentConversationId}`);
const copies: ForkDocumentCopy[] = [];
const copiedAddresses = new Set<string>();
await collectCopies(
storage,
{ kind: "conversation", conversationId: entry.entry.conversationId },
entry.commitSeq,
"asOf",
childConversationId,
copies,
copiedAddresses,
context,
);
await collectCopies(
storage,
{ kind: "conversation", conversationId: parentConversationId },
"current",
"current",
childConversationId,
copies,
copiedAddresses,
context,
);
return copies;
}
async function collectCopies(
storage: Storage,
scope: Extract<DocumentRecord["scope"], { readonly kind: "conversation" }>,
at: DocumentPoint,
policy: ForkPolicy,
childConversationId: ConversationId,
copies: ForkDocumentCopy[],
copiedAddresses: Set<string>,
context: Context,
): Promise<void> {
let cursor: Cursor | undefined;
do {
const page = await storage.scanDocuments({ scope, at }, SCAN_PAGE_SIZE, cursor, context);
for (const source of page.items) {
if (source.scope.kind !== "conversation" || source.fork !== policy) continue;
const id = await storage.mintId<DocumentId>();
const identity = {
id,
kind: source.kind,
...(source.key === undefined ? {} : { key: source.key }),
scope: { kind: "conversation" as const, conversationId: childConversationId },
};
const record: DocumentCreate =
source.history === "latest"
? { ...identity, history: "latest", fork: source.fork }
: { ...identity, history: "rewindable", fork: source.fork };
const copyAddress = addressId(record);
if (copiedAddresses.has(copyAddress)) {
const member = record.key === undefined ? record.kind : `${record.kind}/${record.key}`;
throw new Error(`Fork selects multiple source documents for ${member}`);
}
copiedAddresses.add(copyAddress);
copies.push({ record, source: { id: source.id, at } });
}
cursor = page.next;
} while (cursor !== undefined);
}

View file

@ -8,14 +8,18 @@ import {
materializeDocument,
resolveAddress,
} from "../documents.ts";
import { StorageRejected } from "../errors.ts";
import { idFromNumber } from "../ids.ts";
import type {
ConversationDocFamilyToken,
ConversationDocToken,
ConversationId,
DocumentAddress,
Id,
EntryId,
JsonObject,
RewindableConversationDocFamilyToken,
RewindableConversationDocToken,
Seq,
Session,
SessionDocFamilyToken,
SessionDocToken,
@ -23,6 +27,7 @@ import type {
StorageWrite,
TaskDocFamilyToken,
TaskDocToken,
TaskId,
Tx,
} from "../types.ts";
import type { CommitChange, CommitPublication } from "./publications.ts";
@ -75,12 +80,12 @@ export class SessionKernel implements Session {
snapshot<T extends JsonObject>(token: SessionDocToken<T>, context: Context): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject>(
token: ConversationDocToken<T>,
conversationId: Id,
conversationId: ConversationId,
context: Context,
): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject>(
token: TaskDocToken<T>,
taskId: Id,
taskId: TaskId,
context: Context,
): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject, I extends JsonValue>(
@ -90,13 +95,13 @@ export class SessionKernel implements Session {
): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject, I extends JsonValue>(
token: ConversationDocFamilyToken<T, I>,
conversationId: Id,
conversationId: ConversationId,
key: string,
context: Context,
): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject, I extends JsonValue>(
token: TaskDocFamilyToken<T, I>,
taskId: Id,
taskId: TaskId,
key: string,
context: Context,
): Promise<Readonly<T> | undefined>;
@ -119,15 +124,15 @@ export class SessionKernel implements Session {
snapshotAsOf<T extends JsonObject>(
token: RewindableConversationDocToken<T>,
conversationId: Id,
at: Id,
conversationId: ConversationId,
at: EntryId,
context: Context,
): Promise<Readonly<T> | undefined>;
snapshotAsOf<T extends JsonObject, I extends JsonValue>(
token: RewindableConversationDocFamilyToken<T, I>,
conversationId: Id,
conversationId: ConversationId,
key: string,
at: Id,
at: EntryId,
context: Context,
): Promise<Readonly<T> | undefined>;
async snapshotAsOf(token: AnyDocToken, ...args: readonly unknown[]): Promise<JsonObject | undefined> {
@ -138,7 +143,11 @@ export class SessionKernel implements Session {
throw new TypeError("Session.snapshotAsOf() requires a conversation document");
}
const conversationId = resolved.address.scope.conversationId;
const at = args[resolved.nextArgument] as Id;
const atValue = args[resolved.nextArgument];
if (typeof atValue !== "number" || !Number.isSafeInteger(atValue)) {
throw new TypeError("Session.snapshotAsOf() requires an entry ID");
}
const at = idFromNumber<EntryId>(atValue);
const context = args[resolved.nextArgument + 1] as Context;
return this.#enqueue(async () => {
this.#assertHealthy();
@ -202,13 +211,14 @@ export class SessionKernel implements Session {
tx.discard();
return result;
}
let seq: number;
let seq: Seq;
try {
// Once admitted, caller cancellation does not interrupt Storage settlement.
seq = await this.#storage.commit(writes, withoutAbortSignal(context));
} catch (error) {
tx.discard();
this.#poison = { error };
// Callback errors never reach this branch; StorageRejected alone guarantees that no batch effect committed.
if (!(error instanceof StorageRejected)) this.#poison = { error };
throw error;
}
let documents: DocumentCommitChange[];
@ -224,7 +234,7 @@ export class SessionKernel implements Session {
}
#publish(
seq: number,
seq: Seq,
writes: readonly StorageWrite[],
documents: readonly DocumentCommitChange[],
context: Context,

View file

@ -7,21 +7,28 @@ import {
checkRecordScope,
checkRecordVersion,
documentCreate,
materializeDocumentValue,
type ResolvedAddress,
resolveAddress,
} from "../documents.ts";
import { ReadAfterWrite } from "../errors.ts";
import type {
ConversationDocFamilyToken,
ConversationDocToken,
ConversationId,
ConversationOwnership,
ConversationQuery,
ConversationRecord,
Cursor,
DocumentAddress,
DocumentCopySource,
DocumentCreate,
DocumentId,
DocumentRecord,
EntryDraft,
EntryId,
EntryQuery,
EntryRecord,
Id,
JsonObject,
Seq,
SessionDocFamilyToken,
@ -31,12 +38,13 @@ import type {
Task,
TaskDocFamilyToken,
TaskDocToken,
TaskId,
TaskOptions,
TaskQuery,
TaskRecord,
TaskRef,
Tx,
} from "../types.ts";
import { prepareForkDocumentCopies } from "./forks.ts";
type AnyTaskRecord = TaskRecord<JsonValue, JsonValue, JsonValue>;
@ -44,25 +52,27 @@ const INTERNAL_SCAN_PAGE_SIZE = 256;
const EMPTY_OPERATIONS: readonly Op[] = [];
const TABLE_JSON_COPY_OPTIONS = { omitUndefinedProperties: true } as const;
/** A transaction read a table after its first table write. Read every required row before writing. */
export class ReadAfterWrite extends Error {
constructor(method: string) {
super(`Tx.${method}() cannot read tables after the first table write`);
this.name = "ReadAfterWrite";
}
}
/** Committed change of one document incarnation. */
export type DocumentCommitChange = {
readonly type: "document";
readonly record: DocumentRecord;
/** Conversation owning the document; task documents derive it from their task record. */
readonly conversationId: Id | undefined;
/** Exact adopted immutable revision, or `null` when this commit retired the incarnation. */
readonly value: JsonObject | null;
/** Exact adopted operations for an ordinary update; empty for creation and retirement. */
readonly ops: readonly Op[];
};
export type DocumentCommitChange =
| {
readonly type: "document";
readonly record: DocumentRecord;
/** Conversation owning the document; task documents derive it from their task record. Undefined only for Session documents. */
readonly conversationId: ConversationId | undefined;
/** Definition version of `value`; absent when this commit retired the incarnation. */
readonly version: number | undefined;
/** Exact adopted immutable revision, or `null` when this commit retired the incarnation. */
readonly value: JsonObject | null;
/** Exact adopted operations for an ordinary update; empty for creation and retirement. */
readonly ops: readonly Op[];
}
| {
/** Definition-free child initialization; consumers hydrate through a source or watch. */
readonly type: "document.copy";
readonly record: DocumentRecord;
readonly conversationId: ConversationId;
readonly source: DocumentCopySource;
};
/** One committed document incarnation owned by the Session tracker cache. */
export type LoadedDocument = {
@ -88,14 +98,14 @@ export interface TransactionHost {
/** Install a newly committed incarnation. */
install(document: LoadedDocument): void;
/** Remove a retired incarnation if it is still the cached occupant of its address. */
evict(addressId: string, recordId: Id): void;
evict(addressId: string, recordId: DocumentId): void;
}
/** Committed and candidate state for one task touched by this transaction. */
type TransactionTask = {
committedRead?: Promise<AnyTaskRecord | undefined>;
write?: { readonly kind: "create" | "replace"; readonly record: AnyTaskRecord };
publicationConversationId?: Id;
publicationConversationId?: ConversationId;
};
/** Storage/cache provenance of one staged document incarnation. */
@ -107,14 +117,19 @@ type DocumentTarget =
readonly version: number;
readonly tracker: Tracker<JsonObject>;
}
| {
readonly kind: "fork-copy";
readonly record: DocumentCreate;
readonly source: DocumentCopySource;
}
| { readonly kind: "retire-only"; readonly record: DocumentRecord };
/** One document incarnation acquired, created, or retired by this transaction. */
type DocumentEntry = {
readonly addressId: string;
readonly address: DocumentAddress;
/** Absent only for retirement entries discovered by a terminal-task scan. */
readonly definition?: AnyDocDefinition;
/** Absent for definition-free fork copies and retirement entries discovered by a terminal-task scan. */
definition?: AnyDocDefinition;
/** Memoized public acquisition; absent for metadata-only retirement. */
draftPromise?: Promise<Draft<JsonObject>>;
/** Set after acquisition or retirement lookup finds the affected incarnation. */
@ -123,7 +138,7 @@ type DocumentEntry = {
prepared?: Prepared<JsonObject>;
retireOnCommit: boolean;
/** Resolved before Storage admission so adoption performs no reads. */
conversationId?: Id;
conversationId?: ConversationId;
};
/**
@ -141,9 +156,11 @@ export class Transaction implements Tx {
/** Atomic batch; conversation and entry writes stage eagerly, while task and document writes assemble later. */
readonly #writes: StorageWrite[] = [];
readonly #createdConversationIds = new Set<Id>();
readonly #createdConversationIds = new Set<ConversationId>();
readonly #forkSourceConversationIds = new Set<ConversationId>();
readonly #forkSourceDocumentIds = new Set<DocumentId>();
/** One entry per task touched by a public read, candidate write, or document-owner lookup. */
readonly #tasksById = new Map<Id, TransactionTask>();
readonly #tasksById = new Map<TaskId, TransactionTask>();
/** Every document acquisition or retirement marker in staging order. */
readonly #documents: DocumentEntry[] = [];
@ -157,20 +174,22 @@ export class Transaction implements Tx {
// ─── Table reads ────────────────────────────────────────────────────────
conversation(id: Id): Promise<ConversationRecord | undefined> {
conversation(id: ConversationId): Promise<ConversationRecord | undefined> {
return this.#read("conversation", () => this.#host.storage.conversation(id, this.#context));
}
entry(id: Id): Promise<EntryRecord | undefined> {
entry(id: EntryId): Promise<EntryRecord | undefined> {
return this.#read("entry", async () => (await this.#host.storage.entry(id, this.#context))?.entry);
}
task(id: Id): Promise<AnyTaskRecord | undefined> {
task(id: TaskId): Promise<AnyTaskRecord | undefined> {
return this.#read("task", () => this.#committedTask(id));
}
scanConversations(limit: number, cursor?: Cursor) {
return this.#read("scanConversations", () => this.#host.storage.scanConversations(limit, cursor, this.#context));
scanConversations(query: ConversationQuery, limit: number, cursor?: Cursor) {
return this.#read("scanConversations", () =>
this.#host.storage.scanConversations(query, limit, cursor, this.#context),
);
}
scanEntries(query: EntryQuery, limit: number, cursor?: Cursor) {
@ -183,22 +202,66 @@ export class Transaction implements Tx {
// ─── Table writes ───────────────────────────────────────────────────────
createConversation(value: Omit<ConversationRecord, "id">): Promise<ConversationRecord> {
return this.#write(async () => {
const id = await this.#host.storage.mintId();
this.#assertOpen();
const record = copyJson({ ...value, id }, TABLE_JSON_COPY_OPTIONS) as ConversationRecord;
this.#createdConversationIds.add(id);
this.#writes.push({ type: "conversation", value: record });
return record;
});
createConversation(options: { readonly ownership: ConversationOwnership }): Promise<ConversationRecord> {
return this.#write(() => this.#stageConversation(undefined, options.ownership));
}
appendEntry(conversationId: Id, value: EntryDraft): Promise<EntryRecord> {
forkConversation(
parentConversationId: ConversationId,
at: EntryId,
options: { readonly ownership: ConversationOwnership },
): Promise<ConversationRecord> {
return this.#write(() =>
this.#stageConversation({ conversationId: parentConversationId, at }, options.ownership),
);
}
async #stageConversation(
parent: NonNullable<ConversationRecord["parent"]> | undefined,
ownership: ConversationOwnership,
): Promise<ConversationRecord> {
const ownerTaskId = ownership.kind === "task" ? ownership.taskId : undefined;
const id = await this.#host.storage.mintId<ConversationId>();
this.#assertOpen();
let owner: ConversationRecord["owner"];
if (ownerTaskId !== undefined) {
const task = await this.#currentTask(ownerTaskId);
this.#assertOpen();
if (task === undefined) throw new Error(`Conversation owner task ${ownerTaskId} does not exist`);
owner = { conversationId: task.conversationId, taskId: ownerTaskId };
}
const record: ConversationRecord = {
id,
...(parent === undefined ? {} : { parent }),
...(owner === undefined ? {} : { owner }),
};
const copies =
parent === undefined
? []
: await prepareForkDocumentCopies(this.#host.storage, parent.conversationId, parent.at, id, this.#context);
this.#assertOpen();
for (const copy of copies) {
this.#forkSourceDocumentIds.add(copy.source.id);
const entry: DocumentEntry = {
addressId: addressId(copy.record),
address: copy.record,
target: { kind: "fork-copy", ...copy },
retireOnCommit: false,
};
this.#documents.push(entry);
this.#latestDocumentByAddress.set(entry.addressId, entry);
}
if (parent !== undefined) this.#forkSourceConversationIds.add(parent.conversationId);
this.#createdConversationIds.add(id);
this.#writes.push({ type: "conversation", value: record });
return record;
}
appendEntry(conversationId: ConversationId, value: EntryDraft): Promise<EntryRecord> {
return this.#write(async () => {
await this.#requireConversation(conversationId);
this.#assertOpen();
const id = await this.#host.storage.mintId();
const id = await this.#host.storage.mintId<EntryId>();
this.#assertOpen();
const { head, ...rest } = value;
const record = copyJson(
@ -216,7 +279,7 @@ export class Transaction implements Tx {
task: Task<I, S, R, H>,
input: I,
options?: TaskOptions,
): Promise<TaskRef<R>> {
): Promise<TaskId<R>> {
return this.#write(async () => {
const conversationId = options?.conversationId;
if (conversationId === undefined) throw new TypeError("Tx.createTask() requires options.conversationId");
@ -224,7 +287,7 @@ export class Transaction implements Tx {
this.#assertOpen();
const definition = task.definition;
const checkpoint = definition.initial(input);
const id = await this.#host.storage.mintId();
const id = await this.#host.storage.mintId<TaskId<R>>();
this.#assertOpen();
const record = copyJson(
{
@ -241,7 +304,7 @@ export class Transaction implements Tx {
TABLE_JSON_COPY_OPTIONS,
) as unknown as AnyTaskRecord;
this.#tasksById.set(id, { write: { kind: "create", record } });
return { id };
return id;
});
}
@ -249,9 +312,13 @@ export class Transaction implements Tx {
this.#assertOpen();
this.#hasTableWrite = true;
const task = this.#taskEntry(value.id);
if (task.write?.record.state.status === "terminal") {
const candidate = task.write?.record;
if (candidate?.state.status === "terminal") {
throw new Error(`Task ${value.id} already has a terminal candidate`);
}
if (candidate !== undefined && candidate.conversationId !== value.conversationId) {
throw new Error(`Task ${value.id} cannot change conversations`);
}
task.write = {
kind: task.write?.kind === "create" ? "create" : "replace",
record: copyJson(value, TABLE_JSON_COPY_OPTIONS) as unknown as AnyTaskRecord,
@ -260,8 +327,8 @@ export class Transaction implements Tx {
// ─── Documents ──────────────────────────────────────────────────────────
doc<T extends JsonObject>(token: SessionDocToken<T>): Promise<Draft<T>>;
doc<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: Id): Promise<Draft<T>>;
doc<T extends JsonObject>(token: TaskDocToken<T>, taskId: Id): Promise<Draft<T>>;
doc<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: ConversationId): Promise<Draft<T>>;
doc<T extends JsonObject>(token: TaskDocToken<T>, taskId: TaskId): Promise<Draft<T>>;
doc<T extends JsonObject, I extends JsonValue>(
token: SessionDocFamilyToken<T, I>,
key: string,
@ -269,13 +336,13 @@ export class Transaction implements Tx {
): Promise<Draft<T>>;
doc<T extends JsonObject, I extends JsonValue>(
token: ConversationDocFamilyToken<T, I>,
conversationId: Id,
conversationId: ConversationId,
key: string,
seed: I,
): Promise<Draft<T>>;
doc<T extends JsonObject, I extends JsonValue>(
token: TaskDocFamilyToken<T, I>,
taskId: Id,
taskId: TaskId,
key: string,
seed: I,
): Promise<Draft<T>>;
@ -286,8 +353,12 @@ export class Transaction implements Tx {
const resolved = resolveAddress(definition, args);
this.#assertTaskDocumentsOpen(resolved);
const latest = this.#latestDocumentByAddress.get(resolved.id);
if (latest !== undefined && !latest.retireOnCommit && latest.draftPromise !== undefined) {
return latest.draftPromise;
if (latest !== undefined && !latest.retireOnCommit) {
if (latest.draftPromise !== undefined) return latest.draftPromise;
if (latest.target?.kind === "fork-copy") {
latest.draftPromise = this.#track(this.#acquireForkCopy(latest, definition, latest.target));
return latest.draftPromise;
}
}
const seed = definition.family === true ? copyJson(args[resolved.nextArgument]) : undefined;
const docEntry: DocumentEntry = {
@ -307,17 +378,17 @@ export class Transaction implements Tx {
}
retireDoc<T extends JsonObject>(token: SessionDocToken<T>): Promise<void>;
retireDoc<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: Id): Promise<void>;
retireDoc<T extends JsonObject>(token: TaskDocToken<T>, taskId: Id): Promise<void>;
retireDoc<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: ConversationId): Promise<void>;
retireDoc<T extends JsonObject>(token: TaskDocToken<T>, taskId: TaskId): Promise<void>;
retireDoc<T extends JsonObject, I extends JsonValue>(token: SessionDocFamilyToken<T, I>, key: string): Promise<void>;
retireDoc<T extends JsonObject, I extends JsonValue>(
token: ConversationDocFamilyToken<T, I>,
conversationId: Id,
conversationId: ConversationId,
key: string,
): Promise<void>;
retireDoc<T extends JsonObject, I extends JsonValue>(
token: TaskDocFamilyToken<T, I>,
taskId: Id,
taskId: TaskId,
key: string,
): Promise<void>;
retireDoc(token: AnyDocToken, ...args: readonly unknown[]): Promise<void> {
@ -327,6 +398,11 @@ export class Transaction implements Tx {
const resolved = resolveAddress(definition, args);
const latest = this.#latestDocumentByAddress.get(resolved.id);
if (latest?.retireOnCommit) return Promise.resolve();
if (latest?.target?.kind === "fork-copy") {
checkRecordScope(definition, latest.target.record);
latest.retireOnCommit = true;
return Promise.resolve();
}
if (latest?.draftPromise !== undefined) {
// Retirement of an acquired draft persists its final content before retirement.
latest.retireOnCommit = true;
@ -370,7 +446,7 @@ export class Transaction implements Tx {
const value = copyJson(
definition.family === true ? definition.initial(seed) : definition.initial(),
) as JsonObject;
const id = await this.#host.storage.mintId();
const id = await this.#host.storage.mintId<DocumentId>();
this.#assertOpen();
const tracker = track(value);
entry.target = {
@ -383,6 +459,33 @@ export class Transaction implements Tx {
return entry.change.state;
}
async #acquireForkCopy(
entry: DocumentEntry,
definition: AnyDocDefinition,
target: Extract<DocumentTarget, { readonly kind: "fork-copy" }>,
): Promise<Draft<JsonObject>> {
const stored = await this.#host.storage.document(target.source.id, target.source.at, this.#context);
this.#assertOpen();
if (stored === undefined) {
throw new Error(`Fork source document ${target.source.id} cannot be read`);
}
if (
stored.record.scope.kind !== "conversation" ||
stored.record.kind !== target.record.kind ||
stored.record.key !== target.record.key ||
stored.record.history !== target.record.history ||
stored.record.fork !== target.record.fork
) {
throw new Error(`Fork source document ${target.source.id} does not match the copied record`);
}
const value = materializeDocumentValue(definition, target.record, stored.version, stored.value);
const tracker = track(value);
entry.definition = definition;
entry.target = { kind: "created", record: target.record, version: definition.version, tracker };
entry.change = tracker.beginChange();
return entry.change.state;
}
async #findRetirement(entry: DocumentEntry): Promise<void> {
const record =
this.#host.cached(entry.addressId)?.record ??
@ -456,11 +559,35 @@ export class Transaction implements Tx {
type: "document",
record,
conversationId: document.conversationId,
version: document.retireOnCommit ? undefined : target.version,
value: document.retireOnCommit ? null : prepared.value,
ops: EMPTY_OPERATIONS,
});
break;
}
case "fork-copy": {
const record: DocumentRecord = document.retireOnCommit
? { ...target.record, createdAt: seq, retiredAt: seq }
: { ...target.record, createdAt: seq };
if (document.retireOnCommit) {
publications.push({
type: "document",
record,
conversationId: document.conversationId,
version: undefined,
value: null,
ops: EMPTY_OPERATIONS,
});
} else {
publications.push({
type: "document.copy",
record,
conversationId: document.conversationId!,
source: target.source,
});
}
break;
}
case "loaded": {
const prepared = document.prepared!;
const changed = prepared.ops.length > 0;
@ -479,6 +606,7 @@ export class Transaction implements Tx {
? { ...target.document.record, retiredAt: seq }
: target.document.record,
conversationId: document.conversationId,
version: document.retireOnCommit ? undefined : document.definition!.version,
value: document.retireOnCommit ? null : prepared.value,
ops: document.retireOnCommit ? EMPTY_OPERATIONS : prepared.ops,
});
@ -490,6 +618,7 @@ export class Transaction implements Tx {
type: "document",
record: { ...target.record, retiredAt: seq },
conversationId: document.conversationId,
version: undefined,
value: null,
ops: EMPTY_OPERATIONS,
});
@ -501,15 +630,20 @@ export class Transaction implements Tx {
async #assemble(): Promise<StorageWrite[]> {
const storage = this.#host.storage;
this.#rejectForkSourceWrites();
await this.#validateConversationOwners();
for (const [id, task] of this.#tasksById) {
if (task.write?.kind !== "replace") continue;
const committed = await this.#committedTask(id);
if (committed === undefined) throw new Error(`Task ${id} does not exist`);
if (committed.state.status === "terminal") throw new Error(`Task ${id} is already terminal`);
if (committed.conversationId !== task.write.record.conversationId) {
throw new Error(`Task ${id} cannot change conversations`);
}
}
// Terminal settlement retires every task document, including ones created by this transaction.
let terminalTaskIds: Set<Id> | undefined;
let terminalTaskIds: Set<TaskId> | undefined;
for (const task of this.#tasksById.values()) {
const candidate = task.write?.record;
if (candidate?.state.status !== "terminal") continue;
@ -517,7 +651,7 @@ export class Transaction implements Tx {
terminalTaskIds.add(candidate.id);
}
if (terminalTaskIds !== undefined) {
const targetedDocumentIds = new Set<Id>();
const targetedDocumentIds = new Set<DocumentId>();
for (const document of this.#documents) {
const scope = document.address.scope;
if (scope.kind !== "task" || !terminalTaskIds.has(scope.taskId)) continue;
@ -583,6 +717,10 @@ export class Transaction implements Tx {
});
if (document.retireOnCommit) writes.push({ type: "document.retire", id: target.record.id });
break;
case "fork-copy":
writes.push({ type: "document.copy", record: target.record, source: target.source });
if (document.retireOnCommit) writes.push({ type: "document.retire", id: target.record.id });
break;
case "loaded": {
const definition = document.definition!;
const prepared = document.prepared!;
@ -617,6 +755,52 @@ export class Transaction implements Tx {
// ─── Helpers ────────────────────────────────────────────────────────────
async #validateConversationOwners(): Promise<void> {
for (const write of this.#writes) {
if (write.type !== "conversation" || write.value.owner === undefined) continue;
const owner = write.value.owner;
const task = await this.#currentTask(owner.taskId);
if (task === undefined) throw new Error(`Conversation owner task ${owner.taskId} does not exist`);
if (task.conversationId !== owner.conversationId) {
throw new Error(`Conversation owner task ${owner.taskId} changed conversations`);
}
if (task.state.status === "terminal") {
throw new Error(`Conversation owner task ${owner.taskId} is terminal`);
}
if (task.abortRequested) {
throw new Error(`Conversation owner task ${owner.taskId} is abort-marked`);
}
}
}
#rejectForkSourceWrites(): void {
for (const document of this.#documents) {
const target = document.target;
if (target === undefined) continue;
const writes =
target.kind !== "loaded" ||
document.retireOnCommit ||
target.document.storedVersion < document.definition!.version ||
document.prepared!.ops.length > 0;
if (!writes) continue;
const record = target.kind === "loaded" ? target.document.record : target.record;
if (this.#forkSourceDocumentIds.has(record.id)) {
throw new Error(`Cannot change fork source document ${record.id} in the fork transaction`);
}
const scope = document.address.scope;
if (
scope.kind === "conversation" &&
this.#forkSourceConversationIds.has(scope.conversationId) &&
record.scope.kind === "conversation" &&
record.fork === "current"
) {
throw new Error(
`Cannot fork conversation ${scope.conversationId} while changing its current-policy documents`,
);
}
}
}
#abortChanges(): void {
for (const document of this.#documents) document.change?.abort();
}
@ -666,14 +850,14 @@ export class Transaction implements Tx {
}
}
async #requireConversation(id: Id): Promise<void> {
async #requireConversation(id: ConversationId): Promise<void> {
if (this.#createdConversationIds.has(id)) return;
if ((await this.#host.storage.conversation(id, this.#context)) === undefined) {
throw new Error(`Conversation ${id} does not exist`);
}
}
#taskEntry(id: Id): TransactionTask {
#taskEntry(id: TaskId): TransactionTask {
let task = this.#tasksById.get(id);
if (task === undefined) {
task = {};
@ -683,11 +867,11 @@ export class Transaction implements Tx {
}
/** Latest candidate task record, falling back to committed state; not a caller table read. */
async #currentTask(id: Id): Promise<AnyTaskRecord | undefined> {
async #currentTask(id: TaskId): Promise<AnyTaskRecord | undefined> {
return this.#tasksById.get(id)?.write?.record ?? (await this.#committedTask(id));
}
#committedTask(id: Id): Promise<AnyTaskRecord | undefined> {
#committedTask(id: TaskId): Promise<AnyTaskRecord | undefined> {
const task = this.#taskEntry(id);
task.committedRead ??= this.#host.storage.task(id, this.#context);
return task.committedRead;

View file

@ -1,17 +1,24 @@
import type { Context, JsonValue } from "@earendil-works/chord";
import type { FileError, FileSystem } from "../../env/index.ts";
import { idFromNumber, seqFromNumber } from "../../ids.ts";
import type {
ConversationId,
ConversationQuery,
Cursor,
DocumentAddress,
DocumentContent,
DocumentCreate,
DocumentId,
DocumentPoint,
DocumentQuery,
EntryId,
EntryQuery,
Id,
Seq,
Storage,
StorageWrite,
SubmissionId,
TaskId,
TaskQuery,
TaskRecord,
} from "../../types.ts";
@ -27,9 +34,9 @@ type StoredTask = TaskRecord<JsonValue, JsonValue, JsonValue>;
type MainOperation =
| Extract<StorageWrite, { readonly type: "conversation" | "entry" | "submission" | "document.retire" }>
| { readonly type: "task"; readonly value: StoredTask }
| { readonly type: "task.sidecar"; readonly id: Id; readonly ordinal: number }
| { readonly type: "task.sidecar"; readonly id: TaskId; readonly ordinal: number }
| { readonly type: "document.create"; readonly record: DocumentCreate; readonly ordinal: number }
| { readonly type: "document.change"; readonly id: Id; readonly ordinal: number };
| { readonly type: "document.change"; readonly id: DocumentId; readonly ordinal: number };
type MainMarker = {
readonly format: typeof FORMAT_VERSION;
@ -40,7 +47,7 @@ type MainMarker = {
type SidecarPayload =
| { readonly type: "task"; readonly value: StoredTask }
| { readonly type: "document"; readonly id: Id; readonly content: DocumentContent };
| { readonly type: "document"; readonly id: DocumentId; readonly content: DocumentContent };
type SidecarRecord = {
readonly format: typeof FORMAT_VERSION;
@ -89,7 +96,7 @@ const isObject = (value: unknown): value is Record<string, unknown> =>
const isSafeInteger = (value: unknown): value is number => Number.isSafeInteger(value);
const sidecarFileName = (kind: "doc" | "task", id: Id): string => `${kind}-${id}.jsonl`;
const sidecarFileName = (kind: "doc" | "task", id: DocumentId | TaskId): string => `${kind}-${id}.jsonl`;
const isSidecarFileName = (name: string): boolean => /^(?:doc|task)-(?:0|[1-9]\d*)\.jsonl$/.test(name);
@ -179,7 +186,7 @@ const parseMainMarker = (text: string, line: number): MainMarker => {
return {
format: FORMAT_VERSION,
type: "commit",
seq: value.seq,
seq: seqFromNumber(value.seq),
writes: value.writes.map((write) => validateMainOperation(write, description)),
};
};
@ -236,8 +243,8 @@ export class JsonlStorage implements Storage {
private readonly mainPath: string;
private readonly fsync: boolean;
private readonly memory = new MemoryStorage();
private readonly currentOnlyDocuments = new Set<Id>();
private readonly liveTaskSidecars = new Set<Id>();
private readonly currentOnlyDocuments = new Set<DocumentId>();
private readonly liveTaskSidecars = new Set<TaskId>();
private closed = false;
private poisonError: JsonlStoragePoisonedError | undefined;
@ -297,27 +304,36 @@ export class JsonlStorage implements Storage {
return seq;
}
async mintId() {
return this.store.mintId();
async mintId<I extends Id<string>>(): Promise<I> {
return this.store.mintId<I>();
}
async conversation(id: Id, context: Context) {
async conversation(id: ConversationId, context: Context) {
return this.store.conversation(id, context);
}
async scanConversations(limit: number, cursor: Cursor | undefined, context: Context) {
return this.store.scanConversations(limit, cursor, context);
async scanConversations(query: ConversationQuery, limit: number, cursor: Cursor | undefined, context: Context) {
return this.store.scanConversations(query, limit, cursor, context);
}
entry(id: Id, context: Context): ReturnType<Storage["entry"]>;
entry(conversationId: Id, id: Id, context: Context): ReturnType<Storage["entry"]>;
async entry(idOrConversationId: Id, idOrContext: Id | Context, context?: Context) {
return context === undefined
? this.store.entry(idOrConversationId, idOrContext as Context)
: this.store.entry(idOrConversationId, idOrContext as Id, context);
entry(id: EntryId, context: Context): ReturnType<Storage["entry"]>;
entry(conversationId: ConversationId, id: EntryId, context: Context): ReturnType<Storage["entry"]>;
async entry(idOrConversationId: EntryId | ConversationId, idOrContext: EntryId | Context, context?: Context) {
if (context === undefined)
return this.store.entry(idFromNumber<EntryId>(idOrConversationId), idOrContext as Context);
if (typeof idOrContext !== "number") throw new TypeError("Storage.entry() requires an entry ID");
return this.store.entry(
idFromNumber<ConversationId>(idOrConversationId),
idFromNumber<EntryId>(idOrContext),
context,
);
}
async findLatestHeadMarker(conversationId: Id, atOrBeforeEntryId: Id | undefined, context: Context) {
async findLatestHeadMarker(
conversationId: ConversationId,
atOrBeforeEntryId: EntryId | undefined,
context: Context,
) {
return this.store.findLatestHeadMarker(conversationId, atOrBeforeEntryId, context);
}
@ -325,7 +341,7 @@ export class JsonlStorage implements Storage {
return this.store.scanEntries(query, limit, cursor, context);
}
async task(id: Id, context: Context) {
async task(id: TaskId, context: Context) {
return this.store.task(id, context);
}
@ -333,11 +349,11 @@ export class JsonlStorage implements Storage {
return this.store.scanTasks(query, limit, cursor, context);
}
async submission(id: Id, context: Context) {
async submission(id: SubmissionId, context: Context) {
return this.store.submission(id, context);
}
async submissionByRequest(conversationId: Id, requestId: string, context: Context) {
async submissionByRequest(conversationId: ConversationId, requestId: string, context: Context) {
return this.store.submissionByRequest(conversationId, requestId, context);
}
@ -345,7 +361,7 @@ export class JsonlStorage implements Storage {
return this.store.findDocument(address, at, context);
}
async document(id: Id, at: DocumentPoint, context: Context) {
async document(id: DocumentId, at: DocumentPoint, context: Context) {
return this.store.document(id, at, context);
}
@ -427,10 +443,10 @@ export class JsonlStorage implements Storage {
}
private planReclamations(writes: readonly StorageWrite[], encoded: EncodedCommit): ReadonlyMap<string, string> {
const createdCurrentOnlyDocuments = new Set<Id>();
const retiredDocuments = new Set<Id>();
const baseDocuments = new Set<Id>();
const finalTasks = new Map<Id, StoredTask>();
const createdCurrentOnlyDocuments = new Set<DocumentId>();
const retiredDocuments = new Set<DocumentId>();
const baseDocuments = new Set<DocumentId>();
const finalTasks = new Map<TaskId, StoredTask>();
for (const write of writes) {
switch (write.type) {
case "document.create":
@ -449,7 +465,7 @@ export class JsonlStorage implements Storage {
}
const replacements = new Map<string, string>();
const isCurrentOnlyDocument = (id: Id): boolean =>
const isCurrentOnlyDocument = (id: DocumentId): boolean =>
this.currentOnlyDocuments.has(id) || createdCurrentOnlyDocuments.has(id);
for (const id of retiredDocuments) {
if (isCurrentOnlyDocument(id)) replacements.set(sidecarFileName("doc", id), "");
@ -559,9 +575,9 @@ export class JsonlStorage implements Storage {
}
}
const currentOnlyDocuments = new Set<Id>();
const retiredDocuments = new Set<Id>();
const finalTaskIsLive = new Map<Id, boolean>();
const currentOnlyDocuments = new Set<DocumentId>();
const retiredDocuments = new Set<DocumentId>();
const finalTaskIsLive = new Map<TaskId, boolean>();
for (const { value: marker } of main.lines) {
for (const operation of marker.writes) {
if (operation.type === "document.create") {
@ -577,7 +593,7 @@ export class JsonlStorage implements Storage {
}
const retiredCurrentOnlyDocuments = new Set([...retiredDocuments].filter((id) => currentOnlyDocuments.has(id)));
const latestBases = new Map<Id, SidecarRecord>();
const latestBases = new Map<DocumentId, SidecarRecord>();
for (const { value: marker } of main.lines) {
for (const operation of marker.writes) {
if (operation.type !== "document.create" && operation.type !== "document.change") continue;
@ -604,7 +620,7 @@ export class JsonlStorage implements Storage {
}
}
const isBeforeLatestBase = (id: Id, seq: Seq, ordinal: number): boolean => {
const isBeforeLatestBase = (id: DocumentId, seq: Seq, ordinal: number): boolean => {
const base = latestBases.get(id);
return base !== undefined && (seq < base.seq || (seq === base.seq && ordinal < base.ordinal));
};
@ -710,19 +726,22 @@ export class JsonlStorage implements Storage {
if (!truncated.ok) throw errorFromFile(`tail truncation of ${file}`, truncated.error);
}
const id = Number(file.slice(file.indexOf("-") + 1, -".jsonl".length));
const numericId = Number(file.slice(file.indexOf("-") + 1, -".jsonl".length));
const confirmedLines = parsed.lines.filter((line) =>
confirmed.has(sidecarKey(file, line.value.seq, line.value.ordinal)),
);
let retainedLines: readonly ParsedLine<SidecarRecord>[] | undefined;
if (file.startsWith("task-") && terminalTasks.has(id)) {
if (file.startsWith("task-") && terminalTasks.has(idFromNumber<TaskId>(numericId))) {
retainedLines = [];
} else if (file.startsWith("doc-") && retiredCurrentOnlyDocuments.has(id)) {
retainedLines = [];
} else if (file.startsWith("doc-") && latestBases.has(id)) {
retainedLines = confirmedLines.filter(
(line) => !isBeforeLatestBase(id, line.value.seq, line.value.ordinal),
);
} else if (file.startsWith("doc-")) {
const documentId = idFromNumber<DocumentId>(numericId);
if (retiredCurrentOnlyDocuments.has(documentId)) {
retainedLines = [];
} else if (latestBases.has(documentId)) {
retainedLines = confirmedLines.filter(
(line) => !isBeforeLatestBase(documentId, line.value.seq, line.value.ordinal),
);
}
}
if (
retainedLines !== undefined &&

View file

@ -1,14 +1,20 @@
import type { Context, JsonValue } from "@earendil-works/chord";
import { applyImmutableBatches, type Op } from "@earendil-works/chord/delta";
import { StorageRejected } from "../errors.ts";
import { idFromNumber, seqFromNumber } from "../ids.ts";
import type {
ConversationId,
ConversationQuery,
ConversationRecord,
Cursor,
DocumentAddress,
DocumentContent,
DocumentCreate,
DocumentId,
DocumentPoint,
DocumentQuery,
DocumentRecord,
EntryId,
EntryQuery,
EntryRecord,
Id,
@ -18,7 +24,9 @@ import type {
Storage,
StorageWrite,
StoredDocument,
SubmissionId,
SubmissionRecord,
TaskId,
TaskQuery,
TaskRecord,
} from "../types.ts";
@ -38,7 +46,7 @@ type DocumentAction = {
};
function* documentDeltaBatches(
id: Id,
id: DocumentId,
version: number,
revisions: readonly DocumentRevision[],
start: number,
@ -53,25 +61,28 @@ function* documentDeltaBatches(
}
type DocumentAddressIndex = {
ids: Id[];
currentId?: Id;
ids: DocumentId[];
currentId?: DocumentId;
};
type State = {
conversations: Map<Id, ConversationRecord>;
conversationIds: Id[];
entries: Map<Id, EntryRecord>;
entryIds: Map<Id, Id[]>;
headEntryIds: Map<Id, Id[]>;
entryCommitSeqs: Map<Id, Seq>;
tasks: Map<Id, StoredTask>;
taskIds: Id[];
taskIdsByStatus: Record<TaskStatus, Id[]>;
submissions: Map<Id, SubmissionRecord>;
submissionIdsByRequest: Map<Id, Map<string, Id>>;
documents: Map<Id, StoredDocumentState>;
recordTypes: Map<Id<string>, TableName>;
conversations: Map<ConversationId, ConversationRecord>;
conversationIds: ConversationId[];
conversationIdsByOwnerConversation: Map<ConversationId, ConversationId[]>;
conversationIdsByOwnerTask: Map<TaskId, ConversationId[]>;
entries: Map<EntryId, EntryRecord>;
entryIds: Map<ConversationId, EntryId[]>;
headEntryIds: Map<ConversationId, EntryId[]>;
entryCommitSeqs: Map<EntryId, Seq>;
tasks: Map<TaskId, StoredTask>;
taskIds: TaskId[];
taskIdsByStatus: Record<TaskStatus, TaskId[]>;
submissions: Map<SubmissionId, SubmissionRecord>;
submissionIdsByRequest: Map<ConversationId, Map<string, SubmissionId>>;
documents: Map<DocumentId, StoredDocumentState>;
documentAddresses: Map<string, DocumentAddressIndex>;
documentIdsByScope: Map<string, Id[]>;
documentIdsByScope: Map<string, DocumentId[]>;
};
const clone = <T>(value: T): T => {
@ -102,10 +113,14 @@ const freeze = <T>(value: T): T => {
return Object.freeze(value);
};
const cursorId = (cursor: Readonly<Record<string, JsonValue>> | undefined): Id | undefined =>
cursor?.after as Id | undefined;
const cursorId = <I extends Id<string>>(cursor: Readonly<Record<string, JsonValue>> | undefined): I | undefined => {
const after = cursor?.after;
if (after === undefined) return undefined;
if (typeof after !== "number" || !Number.isSafeInteger(after)) throw new TypeError("Invalid storage cursor");
return idFromNumber<I>(after);
};
const lowerBound = (ids: readonly Id[], target: Id): number => {
const lowerBound = <I extends Id<string>>(ids: readonly I[], target: number): number => {
let low = 0;
let high = ids.length;
while (low < high) {
@ -116,7 +131,7 @@ const lowerBound = (ids: readonly Id[], target: Id): number => {
return low;
};
const upperBound = (ids: readonly Id[], target: Id): number => {
const upperBound = <I extends Id<string>>(ids: readonly I[], target: number): number => {
let low = 0;
let high = ids.length;
while (low < high) {
@ -127,16 +142,25 @@ const upperBound = (ids: readonly Id[], target: Id): number => {
return low;
};
const insertSorted = (ids: Id[], id: Id): void => {
const insertSorted = <I extends Id<string>>(ids: I[], id: I): void => {
if (ids.length === 0 || ids[ids.length - 1] < id) ids.push(id);
else ids.splice(lowerBound(ids, id), 0, id);
};
const removeSorted = (ids: Id[], id: Id): void => {
const removeSorted = <I extends Id<string>>(ids: I[], id: I): void => {
const index = lowerBound(ids, id);
if (ids[index] === id) ids.splice(index, 1);
};
const insertMapId = <K, I extends Id<string>>(index: Map<K, I[]>, key: K, id: I): void => {
let ids = index.get(key);
if (ids === undefined) {
ids = [];
index.set(key, ids);
}
insertSorted(ids, id);
};
const scopeKey = (scope: DocumentRecord["scope"]): string => {
switch (scope.kind) {
case "session":
@ -166,16 +190,9 @@ const isAliveAt = (record: DocumentRecord, at: DocumentPoint): boolean => {
const isCurrentOnly = (record: DocumentRecord): boolean =>
record.scope.kind !== "conversation" || record.history === "latest";
const tableContaining = (state: State, id: Id): TableName | undefined => {
if (state.conversations.has(id)) return "conversation";
if (state.entries.has(id)) return "entry";
if (state.tasks.has(id)) return "task";
if (state.submissions.has(id)) return "submission";
if (state.documents.has(id)) return "document";
return undefined;
};
const tableContaining = (state: State, id: Id<string>): TableName | undefined => state.recordTypes.get(id);
const page = <T extends { readonly id: Id }>(values: readonly T[], limit: number): Page<T, Cursor> => {
const page = <T extends { readonly id: Id<string> }>(values: readonly T[], limit: number): Page<T, Cursor> => {
const items = values.slice(0, limit);
if (values.length <= limit) return { items: clone(items) };
return { items: clone(items), next: { after: items.at(-1)!.id } };
@ -197,8 +214,11 @@ export interface PreparedMemoryCommit {
*/
export class MemoryStorage implements Storage {
private readonly state: State = {
recordTypes: new Map(),
conversations: new Map(),
conversationIds: [],
conversationIdsByOwnerConversation: new Map(),
conversationIdsByOwnerTask: new Map(),
entries: new Map(),
entryIds: new Map(),
headEntryIds: new Map(),
@ -221,12 +241,12 @@ export class MemoryStorage implements Storage {
}
/** Validate and detach one commit without changing observable state. */
prepareCommit(writes: readonly StorageWrite[], seq: Seq = this.nextSeq): PreparedMemoryCommit {
prepareCommit(writes: readonly StorageWrite[], seq: Seq = seqFromNumber(this.nextSeq)): PreparedMemoryCommit {
this.assertOpen();
if (!Number.isSafeInteger(seq) || seq < this.nextSeq) {
throw new Error(`Commit sequence ${seq} does not strictly increase`);
}
const detachedWrites = freeze(writes.map((write) => clone(write)));
const detachedWrites = freeze(this.resolveDocumentCopies(writes.map((write) => clone(write))));
this.checkGlobalIds(detachedWrites);
const documentActions = this.prepareDocumentActions(detachedWrites);
this.checkDocumentActions(documentActions);
@ -244,19 +264,67 @@ export class MemoryStorage implements Storage {
};
}
private resolveDocumentCopies(writes: StorageWrite[]): StorageWrite[] {
if (!writes.some((write) => write.type === "document.copy")) return writes;
const changedDocumentIds = new Set<DocumentId>();
for (const write of writes) {
if (write.type === "document.create" || write.type === "document.copy") {
changedDocumentIds.add(write.record.id);
} else if (write.type === "document.change" || write.type === "document.retire") {
changedDocumentIds.add(write.id);
}
}
return writes.map((write) => {
if (write.type !== "document.copy") return write;
try {
if (changedDocumentIds.has(write.source.id)) {
throw new Error(`Fork source document ${write.source.id} is changed in the copy batch`);
}
const stored = this.materializeDocument(write.source.id, write.source.at);
if (stored === undefined) throw new Error(`Fork source document ${write.source.id} cannot be read`);
if (
stored.record.scope.kind !== "conversation" ||
write.record.scope.kind !== "conversation" ||
stored.record.kind !== write.record.kind ||
stored.record.key !== write.record.key ||
stored.record.history !== write.record.history ||
stored.record.fork !== write.record.fork
) {
throw new Error(`Fork source document ${write.source.id} does not match the copied record`);
}
return {
type: "document.create",
record: write.record,
content: { kind: "base", version: stored.version, value: stored.value },
};
} catch (error) {
if (error instanceof StorageRejected) throw error;
throw new StorageRejected(`Document copy ${write.record.id} was rejected`, { cause: error });
}
});
}
private applyPreparedCommit(
prepared: readonly StorageWrite[],
documentActions: ReadonlyMap<Id, DocumentAction>,
documentActions: ReadonlyMap<DocumentId, DocumentAction>,
seq: Seq,
): Seq {
for (const write of prepared) {
switch (write.type) {
case "conversation":
case "conversation": {
this.state.recordTypes.set(write.value.id, "conversation");
this.state.conversations.set(write.value.id, write.value);
insertSorted(this.state.conversationIds, write.value.id);
const owner = write.value.owner;
if (owner !== undefined) {
insertMapId(this.state.conversationIdsByOwnerConversation, owner.conversationId, write.value.id);
insertMapId(this.state.conversationIdsByOwnerTask, owner.taskId, write.value.id);
}
this.nextId = Math.max(this.nextId, write.value.id + 1);
break;
}
case "entry": {
this.state.recordTypes.set(write.value.id, "entry");
this.state.entries.set(write.value.id, write.value);
this.state.entryCommitSeqs.set(write.value.id, seq);
let ids = this.state.entryIds.get(write.value.conversationId);
@ -277,6 +345,7 @@ export class MemoryStorage implements Storage {
break;
}
case "task": {
this.state.recordTypes.set(write.value.id, "task");
const previous = this.state.tasks.get(write.value.id);
if (previous === undefined) {
insertSorted(this.state.taskIds, write.value.id);
@ -290,6 +359,7 @@ export class MemoryStorage implements Storage {
break;
}
case "submission": {
this.state.recordTypes.set(write.value.id, "submission");
const previous = this.state.submissions.get(write.value.id);
if (previous?.requestId !== undefined) {
const previousRequests = this.state.submissionIdsByRequest.get(previous.conversationId);
@ -310,6 +380,8 @@ export class MemoryStorage implements Storage {
this.nextId = Math.max(this.nextId, write.value.id + 1);
break;
}
case "document.copy":
throw new Error("Prepared document copy was not resolved");
case "document.create":
case "document.change":
case "document.retire":
@ -322,58 +394,75 @@ export class MemoryStorage implements Storage {
return seq;
}
async mintId(): Promise<Id> {
async mintId<I extends Id<string>>(): Promise<I> {
this.assertOpen();
if (!Number.isSafeInteger(this.nextId)) throw new Error("ID space is exhausted");
return this.nextId++;
return idFromNumber<I>(this.nextId++);
}
async conversation(id: Id, _context: Context): Promise<ConversationRecord | undefined> {
async conversation(id: ConversationId, _context: Context): Promise<ConversationRecord | undefined> {
this.assertOpen();
const value = this.state.conversations.get(id);
return value === undefined ? undefined : clone(value);
}
async scanConversations(
query: ConversationQuery,
limit: number,
cursor: Cursor | undefined,
_context: Context,
): Promise<Page<ConversationRecord, Cursor>> {
this.assertOpen();
const ids =
query.ownerTaskId !== undefined
? (this.state.conversationIdsByOwnerTask.get(query.ownerTaskId) ?? [])
: query.ownerConversationId !== undefined
? (this.state.conversationIdsByOwnerConversation.get(query.ownerConversationId) ?? [])
: this.state.conversationIds;
const after = cursorId(cursor);
const start = after === undefined ? 0 : upperBound(this.state.conversationIds, after);
const values = this.state.conversationIds
.slice(start, start + limit + 1)
.map((id) => this.state.conversations.get(id)!);
const start = after === undefined ? 0 : upperBound(ids, after);
const values: ConversationRecord[] = [];
for (let index = start; index < ids.length && values.length <= limit; index++) {
const value = this.state.conversations.get(ids[index]!)!;
if (query.ownerConversationId !== undefined && value.owner?.conversationId !== query.ownerConversationId) {
continue;
}
values.push(value);
}
return page(values, limit);
}
entry(id: Id, context: Context): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
entry(id: EntryId, context: Context): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
entry(
conversationId: Id,
id: Id,
conversationId: ConversationId,
id: EntryId,
context: Context,
): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
async entry(
idOrConversationId: Id,
idOrContext: Id | Context,
idOrConversationId: EntryId | ConversationId,
idOrContext: EntryId | Context,
context?: Context,
): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined> {
this.assertOpen();
const id = context === undefined ? idOrConversationId : (idOrContext as Id);
const entry =
context === undefined
? this.state.entries.get(id)
: this.visibleEntries(idOrConversationId, id, id).next().value;
if (context === undefined) {
const id = idFromNumber<EntryId>(idOrConversationId);
const entry = this.state.entries.get(id);
if (entry === undefined) return undefined;
return { entry: clone(entry), commitSeq: this.state.entryCommitSeqs.get(id)! };
}
if (typeof idOrContext !== "number") throw new TypeError("Storage.entry() requires an entry ID");
const conversationId = idFromNumber<ConversationId>(idOrConversationId);
const id = idFromNumber<EntryId>(idOrContext);
const entry = this.visibleEntries(conversationId, id, id).next().value;
if (entry === undefined) return undefined;
return { entry: clone(entry), commitSeq: this.state.entryCommitSeqs.get(id)! };
}
async findLatestHeadMarker(
conversationId: Id,
atOrBeforeEntryId: Id | undefined,
conversationId: ConversationId,
atOrBeforeEntryId: EntryId | undefined,
_context: Context,
): Promise<(EntryRecord & { readonly head: Id }) | undefined> {
): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> {
this.assertOpen();
if (!this.state.conversations.has(conversationId)) {
throw new Error(`Unknown conversation: ${conversationId}`);
@ -412,7 +501,7 @@ export class MemoryStorage implements Storage {
return page(visible, limit);
}
async task(id: Id, _context: Context): Promise<StoredTask | undefined> {
async task(id: TaskId, _context: Context): Promise<StoredTask | undefined> {
this.assertOpen();
const value = this.state.tasks.get(id);
return value === undefined ? undefined : clone(value);
@ -440,14 +529,14 @@ export class MemoryStorage implements Storage {
return page(values, limit);
}
async submission(id: Id, _context: Context): Promise<SubmissionRecord | undefined> {
async submission(id: SubmissionId, _context: Context): Promise<SubmissionRecord | undefined> {
this.assertOpen();
const value = this.state.submissions.get(id);
return value === undefined ? undefined : clone(value);
}
async submissionByRequest(
conversationId: Id,
conversationId: ConversationId,
requestId: string,
_context: Context,
): Promise<SubmissionRecord | undefined> {
@ -474,24 +563,12 @@ export class MemoryStorage implements Storage {
return undefined;
}
async document(id: Id, at: DocumentPoint, _context: Context): Promise<StoredDocument | undefined> {
async document(id: DocumentId, at: DocumentPoint, _context: Context): Promise<StoredDocument | undefined> {
this.assertOpen();
const stored = this.state.documents.get(id);
if (stored === undefined) return undefined;
if (at !== "current" && isCurrentOnly(stored.record)) {
throw new Error(`Document ${id} does not retain historical content`);
}
if (!isAliveAt(stored.record, at)) return undefined;
const revisions = at === "current" ? stored.revisions : stored.revisions.filter((revision) => revision.seq <= at);
let baseIndex = revisions.length - 1;
while (baseIndex >= 0 && revisions[baseIndex]!.kind !== "base") baseIndex--;
const base = revisions[baseIndex];
if (base?.kind !== "base") throw new Error(`Document ${id} is missing a required base`);
const value = applyImmutableBatches(
base.value,
documentDeltaBatches(id, base.version, revisions, baseIndex + 1),
) as JsonObject;
return { record: clone(stored.record), version: base.version, value: clone(value) };
const stored = this.materializeDocument(id, at);
return stored === undefined
? undefined
: { record: clone(stored.record), version: stored.version, value: clone(stored.value) };
}
async scanDocuments(
@ -517,10 +594,29 @@ export class MemoryStorage implements Storage {
this.closed = true;
}
private materializeDocument(id: DocumentId, at: DocumentPoint): StoredDocument | undefined {
const stored = this.state.documents.get(id);
if (stored === undefined) return undefined;
if (at !== "current" && isCurrentOnly(stored.record)) {
throw new Error(`Document ${id} does not retain historical content`);
}
if (!isAliveAt(stored.record, at)) return undefined;
const revisions = at === "current" ? stored.revisions : stored.revisions.filter((revision) => revision.seq <= at);
let baseIndex = revisions.length - 1;
while (baseIndex >= 0 && revisions[baseIndex]!.kind !== "base") baseIndex--;
const base = revisions[baseIndex];
if (base?.kind !== "base") throw new Error(`Document ${id} is missing a required base`);
const value = applyImmutableBatches(
base.value,
documentDeltaBatches(id, base.version, revisions, baseIndex + 1),
) as JsonObject;
return { record: stored.record, version: base.version, value };
}
private *visibleEntries(
conversationId: Id,
minEntryId = Number.NEGATIVE_INFINITY,
maxEntryId = Number.POSITIVE_INFINITY,
conversationId: ConversationId,
minEntryId: number = Number.NEGATIVE_INFINITY,
maxEntryId: number = Number.POSITIVE_INFINITY,
): Generator<EntryRecord> {
if (!this.state.conversations.has(conversationId)) {
throw new Error(`Unknown conversation: ${conversationId}`);
@ -543,11 +639,12 @@ export class MemoryStorage implements Storage {
}
private checkGlobalIds(writes: readonly StorageWrite[]): void {
const claimed = new Map<Id, TableName>();
const claimed = new Map<Id<string>, TableName>();
for (const write of writes) {
if (write.type === "document.change" || write.type === "document.retire") continue;
const table: TableName = write.type === "document.create" ? "document" : write.type;
const id = write.type === "document.create" ? write.record.id : write.value.id;
const document = write.type === "document.create" || write.type === "document.copy";
const table: TableName = document ? "document" : write.type;
const id = document ? write.record.id : write.value.id;
const existing = tableContaining(this.state, id);
const earlier = claimed.get(id);
if (table === "conversation" || table === "entry" || table === "document") {
@ -563,8 +660,8 @@ export class MemoryStorage implements Storage {
}
}
private prepareDocumentActions(writes: readonly StorageWrite[]): Map<Id, DocumentAction> {
const actions = new Map<Id, DocumentAction>();
private prepareDocumentActions(writes: readonly StorageWrite[]): Map<DocumentId, DocumentAction> {
const actions = new Map<DocumentId, DocumentAction>();
for (const write of writes) {
if (write.type !== "document.create" && write.type !== "document.change" && write.type !== "document.retire") {
continue;
@ -596,7 +693,7 @@ export class MemoryStorage implements Storage {
return actions;
}
private checkDocumentActions(actions: ReadonlyMap<Id, DocumentAction>): void {
private checkDocumentActions(actions: ReadonlyMap<DocumentId, DocumentAction>): void {
const liveCounts = new Map<string, number>();
for (const [id, action] of actions) {
const existing = this.state.documents.get(id);
@ -625,7 +722,7 @@ export class MemoryStorage implements Storage {
}
}
private applyDocumentActions(actions: ReadonlyMap<Id, DocumentAction>, seq: Seq): void {
private applyDocumentActions(actions: ReadonlyMap<DocumentId, DocumentAction>, seq: Seq): void {
for (const [id, action] of actions) {
let stored = this.state.documents.get(id);
if (action.create !== undefined) {
@ -635,6 +732,7 @@ export class MemoryStorage implements Storage {
...(action.retire ? { retiredAt: seq } : {}),
};
stored = { record, revisions: [{ ...action.content!, seq }] };
this.state.recordTypes.set(id, "document");
this.state.documents.set(id, stored);
const key = recordAddressKey(record);

View file

@ -17,7 +17,11 @@ export interface SqliteStatement {
* Queries and transaction callbacks are synchronous so the same storage core can
* run on Node, Bun, and Cloudflare Durable Object SQLite. An adapter may return a
* promise from `transaction` while it waits for the transaction to settle.
* Callers must not close the database while a returned settlement is pending.
* When the callback throws, the adapter must roll the transaction back before
* rethrowing that same error. If rollback fails, it must throw a different error
* (for example an `AggregateError`) so callers cannot mistake the callback error
* for a guaranteed rollback. Callers must not close the database while a
* returned settlement is pending.
*/
export interface SqliteDatabase {
exec(sql: string): void;

View file

@ -19,8 +19,12 @@ const INITIAL_SCHEMA: readonly string[] = [
) STRICT`,
`CREATE TABLE conversations (
id INTEGER PRIMARY KEY,
owner_conversation_id INTEGER,
owner_task_id INTEGER,
record TEXT NOT NULL CHECK (json_valid(record))
) STRICT`,
"CREATE INDEX conversations_by_owner_conversation ON conversations (owner_conversation_id, id)",
"CREATE INDEX conversations_by_owner_task ON conversations (owner_task_id, id)",
`CREATE TABLE entries (
id INTEGER PRIMARY KEY,
conversation_id INTEGER NOT NULL,
@ -77,7 +81,7 @@ const INITIAL_SCHEMA: readonly string[] = [
"CREATE INDEX document_revisions_by_kind ON document_revisions (document_id, kind, seq DESC)",
];
/** Immutable, ordered schema history. Append new migrations; never edit released ones. */
/** Immutable, ordered schema history. Append new migrations after the initial schema ships. */
export const SQLITE_MIGRATIONS: readonly SqliteMigration[] = [{ version: 1, statements: INITIAL_SCHEMA }];
export const CURRENT_SQLITE_SCHEMA_VERSION = SQLITE_MIGRATIONS.at(-1)?.version ?? 0;

View file

@ -69,8 +69,8 @@ export class NodeSqliteDatabase implements SqliteDatabase {
} catch (error) {
try {
this.database.exec("ROLLBACK");
} catch {
// Preserve the error that caused the transaction to fail.
} catch (rollbackError) {
throw new AggregateError([error, rollbackError], "SQLite transaction failed and rollback failed");
}
throw error;
}

View file

@ -1,14 +1,20 @@
import type { Context, JsonValue } from "@earendil-works/chord";
import { apply, type Op } from "@earendil-works/chord/delta";
import { StorageRejected } from "../../errors.ts";
import { idFromNumber, seqFromNumber } from "../../ids.ts";
import type {
ConversationId,
ConversationQuery,
ConversationRecord,
Cursor,
DocumentAddress,
DocumentContent,
DocumentCreate,
DocumentId,
DocumentPoint,
DocumentQuery,
DocumentRecord,
EntryId,
EntryQuery,
EntryRecord,
Id,
@ -18,7 +24,9 @@ import type {
Storage,
StorageWrite,
StoredDocument,
SubmissionId,
SubmissionRecord,
TaskId,
TaskQuery,
TaskRecord,
} from "../../types.ts";
@ -40,6 +48,7 @@ type IdRow = { readonly id: number };
type MetadataRow = { readonly next_id: string; readonly next_seq: number };
type DocumentAction = {
create?: DocumentCreate;
copy?: Extract<StorageWrite, { readonly type: "document.copy" }>["source"];
content?: DocumentContent;
retire: boolean;
};
@ -59,9 +68,14 @@ const getRow = <T extends object>(statement: SqliteStatement, ...params: SqliteV
const allRows = <T extends object>(statement: SqliteStatement, ...params: SqliteValue[]): T[] =>
statement.all<T>(...params);
const cursorId = (cursor: Cursor | undefined): Id | undefined => cursor?.after as Id | undefined;
const cursorId = <I extends Id<string>>(cursor: Cursor | undefined): I | undefined => {
const after = cursor?.after;
if (after === undefined) return undefined;
if (typeof after !== "number" || !Number.isSafeInteger(after)) throw new TypeError("Invalid storage cursor");
return idFromNumber<I>(after);
};
const page = <T extends { readonly id: Id }>(values: readonly T[], limit: number): Page<T, Cursor> => {
const page = <T extends { readonly id: Id<string> }>(values: readonly T[], limit: number): Page<T, Cursor> => {
const items = values.slice(0, limit);
if (values.length <= limit) return { items };
return { items, next: { after: items.at(-1)!.id } };
@ -101,7 +115,7 @@ const isAliveAt = (record: DocumentRecord, at: DocumentPoint): boolean => {
const isCurrentOnly = (record: DocumentRecord): boolean =>
record.scope.kind !== "conversation" || record.history === "latest";
const writeId = (write: StorageWrite): Id | undefined => {
const writeId = (write: StorageWrite): Id<string> | undefined => {
switch (write.type) {
case "conversation":
case "entry":
@ -109,6 +123,7 @@ const writeId = (write: StorageWrite): Id | undefined => {
case "submission":
return write.value.id;
case "document.create":
case "document.copy":
return write.record.id;
case "document.change":
case "document.retire":
@ -150,10 +165,10 @@ class StatementCachingDatabase implements SqliteDatabase {
/** Portable SQLite implementation of the Pico storage contract. */
export class SqliteStorage implements Storage {
private readonly db: SqliteDatabase;
private nextId: Id;
private nextId: number;
private closed = false;
private constructor(db: SqliteDatabase, nextId: Id) {
private constructor(db: SqliteDatabase, nextId: number) {
this.db = new StatementCachingDatabase(db);
this.nextId = nextId;
}
@ -186,7 +201,7 @@ export class SqliteStorage implements Storage {
this.db.prepare("SELECT next_id, next_seq FROM durable_metadata WHERE singleton = 1"),
);
if (metadata === undefined) throw new Error("Durable SQLite metadata is missing");
const committedSeq = metadata.next_seq;
const committedSeq = seqFromNumber(metadata.next_seq);
this.checkGlobalIds(writes);
this.checkDocumentActions(documentActions);
for (const write of writes) this.applyTableWrite(write, committedSeq);
@ -200,29 +215,39 @@ export class SqliteStorage implements Storage {
return seq;
}
async mintId(): Promise<Id> {
async mintId<I extends Id<string>>(): Promise<I> {
this.assertOpen();
if (!Number.isSafeInteger(this.nextId)) throw new Error("ID space is exhausted");
return this.nextId++;
return idFromNumber<I>(this.nextId++);
}
async conversation(id: Id, _context: Context): Promise<ConversationRecord | undefined> {
async conversation(id: ConversationId, _context: Context): Promise<ConversationRecord | undefined> {
this.assertOpen();
const row = getRow<JsonRow>(this.db.prepare("SELECT record FROM conversations WHERE id = ?"), id);
return row === undefined ? undefined : parseJson<ConversationRecord>(row.record);
}
async scanConversations(
query: ConversationQuery,
limit: number,
cursor: Cursor | undefined,
_context: Context,
): Promise<Page<ConversationRecord, Cursor>> {
this.assertOpen();
const after = cursorId(cursor) ?? -1;
const clauses = ["id > ?"];
const params: SqliteValue[] = [cursorId(cursor) ?? -1];
if (query.ownerConversationId !== undefined) {
clauses.push("owner_conversation_id = ?");
params.push(query.ownerConversationId);
}
if (query.ownerTaskId !== undefined) {
clauses.push("owner_task_id = ?");
params.push(query.ownerTaskId);
}
params.push(limit + 1);
const rows = allRows<JsonRow>(
this.db.prepare("SELECT record FROM conversations WHERE id > ? ORDER BY id LIMIT ?"),
after,
limit + 1,
this.db.prepare(`SELECT record FROM conversations WHERE ${clauses.join(" AND ")} ORDER BY id LIMIT ?`),
...params,
);
return page(
rows.map((row) => parseJson<ConversationRecord>(row.record)),
@ -230,23 +255,31 @@ export class SqliteStorage implements Storage {
);
}
entry(id: Id, context: Context): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
entry(id: EntryId, context: Context): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
entry(
conversationId: Id,
id: Id,
conversationId: ConversationId,
id: EntryId,
context: Context,
): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
async entry(
idOrConversationId: Id,
idOrContext: Id | Context,
idOrConversationId: EntryId | ConversationId,
idOrContext: EntryId | Context,
context?: Context,
): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined> {
this.assertOpen();
let conversation = context === undefined ? undefined : this.readConversation(idOrConversationId);
if (context !== undefined && conversation === undefined) {
throw new Error(`Unknown conversation: ${idOrConversationId}`);
const id =
context === undefined
? idFromNumber<EntryId>(idOrConversationId)
: typeof idOrContext === "number"
? idFromNumber<EntryId>(idOrContext)
: undefined;
if (id === undefined) throw new TypeError("Storage.entry() requires an entry ID");
let conversation: ConversationRecord | undefined;
if (context !== undefined) {
const conversationId = idFromNumber<ConversationId>(idOrConversationId);
conversation = this.readConversation(conversationId);
if (conversation === undefined) throw new Error(`Unknown conversation: ${conversationId}`);
}
const id = context === undefined ? idOrConversationId : (idOrContext as Id);
const row = getRow<EntryJsonRow>(this.db.prepare("SELECT record, commit_seq FROM entries WHERE id = ?"), id);
if (row === undefined) return undefined;
const entry = parseJson<EntryRecord>(row.record);
@ -259,18 +292,18 @@ export class SqliteStorage implements Storage {
}
if (entry.id > upperEntryId) return undefined;
}
return { entry, commitSeq: row.commit_seq };
return { entry, commitSeq: seqFromNumber(row.commit_seq) };
}
async findLatestHeadMarker(
conversationId: Id,
atOrBeforeEntryId: Id | undefined,
conversationId: ConversationId,
atOrBeforeEntryId: EntryId | undefined,
_context: Context,
): Promise<(EntryRecord & { readonly head: Id }) | undefined> {
): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> {
this.assertOpen();
let conversation = this.readConversation(conversationId);
if (conversation === undefined) throw new Error(`Unknown conversation: ${conversationId}`);
let upper = atOrBeforeEntryId;
let upper: number | undefined = atOrBeforeEntryId;
while (true) {
const row =
upper === undefined
@ -287,7 +320,7 @@ export class SqliteStorage implements Storage {
conversation.id,
upper,
);
if (row !== undefined) return parseJson<EntryRecord & { readonly head: Id }>(row.record);
if (row !== undefined) return parseJson<EntryRecord & { readonly head: EntryId }>(row.record);
if (conversation.parent === undefined) return undefined;
upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at);
conversation = this.readConversation(conversation.parent.conversationId)!;
@ -304,7 +337,7 @@ export class SqliteStorage implements Storage {
let conversation = this.readConversation(query.conversationId);
if (conversation === undefined) throw new Error(`Unknown conversation: ${query.conversationId}`);
const after = cursorId(cursor);
let upper = query.maxEntryId;
let upper: number | undefined = query.maxEntryId;
if (after !== undefined) upper = Math.min(upper ?? Number.MAX_SAFE_INTEGER, after - 1);
const values: EntryRecord[] = [];
while (true) {
@ -332,7 +365,7 @@ export class SqliteStorage implements Storage {
return page(values, limit);
}
async task(id: Id, _context: Context): Promise<StoredTask | undefined> {
async task(id: TaskId, _context: Context): Promise<StoredTask | undefined> {
this.assertOpen();
const row = getRow<JsonRow>(this.db.prepare("SELECT record FROM tasks WHERE id = ?"), id);
return row === undefined ? undefined : parseJson<StoredTask>(row.record);
@ -378,14 +411,14 @@ export class SqliteStorage implements Storage {
);
}
async submission(id: Id, _context: Context): Promise<SubmissionRecord | undefined> {
async submission(id: SubmissionId, _context: Context): Promise<SubmissionRecord | undefined> {
this.assertOpen();
const row = getRow<JsonRow>(this.db.prepare("SELECT record FROM submissions WHERE id = ?"), id);
return row === undefined ? undefined : parseJson<SubmissionRecord>(row.record);
}
async submissionByRequest(
conversationId: Id,
conversationId: ConversationId,
requestId: string,
_context: Context,
): Promise<SubmissionRecord | undefined> {
@ -420,38 +453,9 @@ export class SqliteStorage implements Storage {
return row === undefined ? undefined : parseJson<DocumentRecord>(row.record);
}
async document(id: Id, at: DocumentPoint, _context: Context): Promise<StoredDocument | undefined> {
async document(id: DocumentId, at: DocumentPoint, _context: Context): Promise<StoredDocument | undefined> {
this.assertOpen();
const row = getRow<JsonRow>(this.db.prepare("SELECT record FROM documents WHERE id = ?"), id);
if (row === undefined) return undefined;
const record = parseJson<DocumentRecord>(row.record);
if (at !== "current" && isCurrentOnly(record)) {
throw new Error(`Document ${id} does not retain historical content`);
}
if (!isAliveAt(record, at)) return undefined;
const upper = at === "current" ? Number.MAX_SAFE_INTEGER : at;
const base = getRow<RevisionRow>(
this.db.prepare(`SELECT seq, kind, version, content FROM document_revisions
WHERE document_id = ? AND kind = 'base' AND seq <= ? ORDER BY seq DESC LIMIT 1`),
id,
upper,
);
if (base === undefined) throw new Error(`Document ${id} is missing a required base`);
let value = parseJson<JsonObject>(base.content);
const tail = allRows<RevisionRow>(
this.db.prepare(`SELECT seq, kind, version, content FROM document_revisions
WHERE document_id = ? AND seq > ? AND seq <= ? ORDER BY seq`),
id,
base.seq,
upper,
);
for (const revision of tail) {
if (revision.kind !== "delta" || revision.version !== base.version) {
throw new Error(`Document ${id} crosses a stored version boundary without a base`);
}
value = apply(value, parseJson<readonly Op[]>(revision.content)) as JsonObject;
}
return { record, version: base.version, value };
return this.materializeDocument(id, at);
}
async scanDocuments(
@ -491,12 +495,45 @@ export class SqliteStorage implements Storage {
await this.db.close();
}
private readConversation(id: Id): ConversationRecord | undefined {
private readConversation(id: ConversationId): ConversationRecord | undefined {
const row = getRow<JsonRow>(this.db.prepare("SELECT record FROM conversations WHERE id = ?"), id);
return row === undefined ? undefined : parseJson<ConversationRecord>(row.record);
}
private candidateNextId(writes: readonly StorageWrite[]): Id {
private materializeDocument(id: DocumentId, at: DocumentPoint): StoredDocument | undefined {
const row = getRow<JsonRow>(this.db.prepare("SELECT record FROM documents WHERE id = ?"), id);
if (row === undefined) return undefined;
const record = parseJson<DocumentRecord>(row.record);
if (at !== "current" && isCurrentOnly(record)) {
throw new Error(`Document ${id} does not retain historical content`);
}
if (!isAliveAt(record, at)) return undefined;
const upper = at === "current" ? Number.MAX_SAFE_INTEGER : at;
const base = getRow<RevisionRow>(
this.db.prepare(`SELECT seq, kind, version, content FROM document_revisions
WHERE document_id = ? AND kind = 'base' AND seq <= ? ORDER BY seq DESC LIMIT 1`),
id,
upper,
);
if (base === undefined) throw new Error(`Document ${id} is missing a required base`);
let value = parseJson<JsonObject>(base.content);
const tail = allRows<RevisionRow>(
this.db.prepare(`SELECT seq, kind, version, content FROM document_revisions
WHERE document_id = ? AND seq > ? AND seq <= ? ORDER BY seq`),
id,
base.seq,
upper,
);
for (const revision of tail) {
if (revision.kind !== "delta" || revision.version !== base.version) {
throw new Error(`Document ${id} crosses a stored version boundary without a base`);
}
value = apply(value, parseJson<readonly Op[]>(revision.content)) as JsonObject;
}
return { record, version: base.version, value };
}
private candidateNextId(writes: readonly StorageWrite[]): number {
let nextId = this.nextId;
for (const write of writes) {
const id = writeId(write);
@ -506,12 +543,13 @@ export class SqliteStorage implements Storage {
}
private checkGlobalIds(writes: readonly StorageWrite[]): void {
const claimed = new Map<Id, TableName>();
const claimed = new Map<Id<string>, TableName>();
const lookup = this.db.prepare("SELECT record_type FROM record_ids WHERE id = ?");
for (const write of writes) {
if (write.type === "document.change" || write.type === "document.retire") continue;
const table: TableName = write.type === "document.create" ? "document" : write.type;
const id = write.type === "document.create" ? write.record.id : write.value.id;
const document = write.type === "document.create" || write.type === "document.copy";
const table: TableName = document ? "document" : write.type;
const id = document ? write.record.id : write.value.id;
const existing = getRow<RecordIdRow>(lookup, id)?.record_type;
const earlier = claimed.get(id);
if (table === "conversation" || table === "entry" || table === "document") {
@ -526,13 +564,18 @@ export class SqliteStorage implements Storage {
}
}
private prepareDocumentActions(writes: readonly StorageWrite[]): Map<Id, DocumentAction> {
const actions = new Map<Id, DocumentAction>();
private prepareDocumentActions(writes: readonly StorageWrite[]): Map<DocumentId, DocumentAction> {
const actions = new Map<DocumentId, DocumentAction>();
for (const write of writes) {
if (write.type !== "document.create" && write.type !== "document.change" && write.type !== "document.retire") {
if (
write.type !== "document.create" &&
write.type !== "document.copy" &&
write.type !== "document.change" &&
write.type !== "document.retire"
) {
continue;
}
const id = write.type === "document.create" ? write.record.id : write.id;
const id = write.type === "document.create" || write.type === "document.copy" ? write.record.id : write.id;
let action = actions.get(id);
if (action === undefined) {
action = { retire: false };
@ -540,14 +583,23 @@ export class SqliteStorage implements Storage {
}
switch (write.type) {
case "document.create":
if (action.create !== undefined || action.content !== undefined) {
if (action.create !== undefined || action.content !== undefined || action.copy !== undefined) {
throw new Error(`Document ${id} has more than one content command`);
}
action.create = write.record;
action.content = write.content;
break;
case "document.copy":
if (action.create !== undefined || action.content !== undefined || action.copy !== undefined) {
throw new Error(`Document ${id} has more than one content command`);
}
action.create = write.record;
action.copy = write.source;
break;
case "document.change":
if (action.content !== undefined) throw new Error(`Document ${id} has more than one content command`);
if (action.content !== undefined || action.copy !== undefined) {
throw new Error(`Document ${id} has more than one content command`);
}
action.content = write.content;
break;
case "document.retire":
@ -559,9 +611,12 @@ export class SqliteStorage implements Storage {
return actions;
}
private checkDocumentActions(actions: ReadonlyMap<Id, DocumentAction>): void {
private checkDocumentActions(actions: ReadonlyMap<DocumentId, DocumentAction>): void {
const liveCounts = new Map<string, number>();
for (const [id, action] of actions) {
if (action.copy !== undefined && actions.has(action.copy.id)) {
throw new StorageRejected(`Document copy ${id} source is changed in the copy batch`);
}
const row = getRow<JsonRow>(this.db.prepare("SELECT record FROM documents WHERE id = ?"), id);
const existing = row === undefined ? undefined : parseJson<DocumentRecord>(row.record);
if (action.create === undefined && existing === undefined) throw new Error(`Unknown document: ${id}`);
@ -592,9 +647,9 @@ export class SqliteStorage implements Storage {
}
}
private currentDocumentId(address: DocumentAddress | DocumentCreate | DocumentRecord): Id | undefined {
private currentDocumentId(address: DocumentAddress | DocumentCreate | DocumentRecord): DocumentId | undefined {
const parts = addressParts(address);
return getRow<IdRow>(
const id = getRow<IdRow>(
this.db.prepare(`SELECT id FROM documents
WHERE kind = ? AND scope_kind = ? AND owner_id = ? AND family = ? AND key_value = ? AND retired_at IS NULL
LIMIT 1`),
@ -604,6 +659,7 @@ export class SqliteStorage implements Storage {
parts.family,
parts.keyValue,
)?.id;
return id === undefined ? undefined : idFromNumber<DocumentId>(id);
}
private applyTableWrite(write: StorageWrite, seq: Seq): void {
@ -611,8 +667,15 @@ export class SqliteStorage implements Storage {
case "conversation":
this.claimId(write.value.id, "conversation");
this.db
.prepare("INSERT INTO conversations (id, record) VALUES (?, ?)")
.run(write.value.id, encodeJson(write.value));
.prepare(
"INSERT INTO conversations (id, owner_conversation_id, owner_task_id, record) VALUES (?, ?, ?, ?)",
)
.run(
write.value.id,
write.value.owner?.conversationId ?? null,
write.value.owner?.taskId ?? null,
encodeJson(write.value),
);
break;
case "entry":
this.claimId(write.value.id, "entry");
@ -652,18 +715,41 @@ export class SqliteStorage implements Storage {
);
break;
case "document.create":
case "document.copy":
case "document.change":
case "document.retire":
break;
}
}
private claimId(id: Id, table: TableName): void {
private claimId(id: Id<string>, table: TableName): void {
this.db.prepare("INSERT OR IGNORE INTO record_ids (id, record_type) VALUES (?, ?)").run(id, table);
}
private applyDocumentActions(actions: ReadonlyMap<Id, DocumentAction>, seq: Seq): void {
private applyDocumentActions(actions: ReadonlyMap<DocumentId, DocumentAction>, seq: Seq): void {
for (const [id, action] of actions) {
let content = action.content;
if (action.copy !== undefined) {
try {
const stored = this.materializeDocument(action.copy.id, action.copy.at);
if (stored === undefined) throw new Error(`Fork source document ${action.copy.id} cannot be read`);
const create = action.create!;
if (
stored.record.scope.kind !== "conversation" ||
create.scope.kind !== "conversation" ||
stored.record.kind !== create.kind ||
stored.record.key !== create.key ||
stored.record.history !== create.history ||
stored.record.fork !== create.fork
) {
throw new Error(`Fork source document ${action.copy.id} does not match the copied record`);
}
content = { kind: "base", version: stored.version, value: stored.value };
} catch (error) {
if (error instanceof StorageRejected) throw error;
throw new StorageRejected(`Document copy ${id} was rejected`, { cause: error });
}
}
let record: DocumentRecord;
if (action.create !== undefined) {
record = {
@ -693,17 +779,16 @@ export class SqliteStorage implements Storage {
record = parseJson<DocumentRecord>(row.record);
}
if (action.content !== undefined) {
if (action.content.kind === "base" && isCurrentOnly(record)) {
if (content !== undefined) {
if (content.kind === "base" && isCurrentOnly(record)) {
this.db.prepare("DELETE FROM document_revisions WHERE document_id = ?").run(id);
}
const encodedContent =
action.content.kind === "base" ? encodeJson(action.content.value) : encodeJson(action.content.ops);
const encodedContent = content.kind === "base" ? encodeJson(content.value) : encodeJson(content.ops);
this.db
.prepare(
"INSERT INTO document_revisions (document_id, seq, kind, version, content) VALUES (?, ?, ?, ?, ?)",
)
.run(id, seq, action.content.kind, action.content.version, encodedContent);
.run(id, seq, content.kind, content.version, encodedContent);
}
if (action.retire) {

View file

@ -1,6 +1,17 @@
import type { JsonValue } from "@earendil-works/chord";
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
import type { DocumentCreate, Id, Seq, Storage, StorageWrite, TaskRecord } from "../types.ts";
import type {
ConversationId,
DocumentCreate,
DocumentId,
EntryId,
Seq,
Storage,
StorageWrite,
SubmissionId,
TaskId,
TaskRecord,
} from "../types.ts";
import { ROOT_CONVERSATION_ID } from "../types.ts";
export type StorageBenchmarkScale = {
@ -43,19 +54,19 @@ export function storageBenchmarkPrimaryRecordCount(scale: StorageBenchmarkScale)
type StoredTask = TaskRecord<JsonValue, JsonValue, JsonValue>;
export type StorageBenchmarkDataset = {
readonly firstEntryId: Id;
readonly firstEntryId: EntryId;
readonly filteredTaskCount: number;
readonly exactDocumentId: Id;
readonly exactDocumentId: DocumentId;
readonly exactDocumentKey: string;
readonly replayDocumentIds: Readonly<Record<(typeof REPLAY_TAILS)[number], Id>>;
readonly historicalDocumentId: Id;
readonly replayDocumentIds: Readonly<Record<(typeof REPLAY_TAILS)[number], DocumentId>>;
readonly historicalDocumentId: DocumentId;
readonly ancientAt: Seq;
readonly recentAt: Seq;
readonly deepestConversationId: Id;
readonly ancestorHeadEntryId: Id;
readonly deepestConversationId: ConversationId;
readonly ancestorHeadEntryId: EntryId;
};
function task(id: Id, index: number): StoredTask {
function task(id: TaskId<JsonValue>, index: number): StoredTask {
const statuses = ["pending", "running", "terminal"] as const;
const status = statuses[index % statuses.length];
const common = {
@ -81,11 +92,11 @@ export async function seedStorageBenchmark(
): Promise<StorageBenchmarkDataset> {
await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT);
let firstEntryId = 0;
let firstEntryId: EntryId | undefined;
for (let start = 0; start < scale.entryCount; start += BATCH_SIZE) {
const writes: StorageWrite[] = [];
for (let index = start; index < Math.min(start + BATCH_SIZE, scale.entryCount); index++) {
const id = await storage.mintId();
const id = await storage.mintId<EntryId>();
if (index === 0) firstEntryId = id;
writes.push({
type: "entry",
@ -104,16 +115,16 @@ export async function seedStorageBenchmark(
for (let start = 0; start < scale.taskCount; start += BATCH_SIZE) {
const writes: StorageWrite[] = [];
for (let index = start; index < Math.min(start + BATCH_SIZE, scale.taskCount); index++) {
writes.push({ type: "task", value: task(await storage.mintId(), index) });
writes.push({ type: "task", value: task(await storage.mintId<TaskId<JsonValue>>(), index) });
}
await storage.commit(writes, BACKGROUND_CONTEXT);
}
let exactDocumentId = 0;
let exactDocumentId: DocumentId | undefined;
for (let start = 0; start < scale.documentCount; start += BATCH_SIZE) {
const writes: StorageWrite[] = [];
for (let index = start; index < Math.min(start + BATCH_SIZE, scale.documentCount); index++) {
const id = await storage.mintId();
const id = await storage.mintId<DocumentId>();
exactDocumentId = id;
writes.push({
type: "document.create",
@ -126,7 +137,7 @@ export async function seedStorageBenchmark(
const replayEntries = await Promise.all(
REPLAY_TAILS.map(async (tail) => {
const id = await storage.mintId();
const id = await storage.mintId<DocumentId>();
return { tail, id };
}),
);
@ -158,7 +169,7 @@ export async function seedStorageBenchmark(
);
}
const historicalDocumentId = await storage.mintId();
const historicalDocumentId = await storage.mintId<DocumentId>();
const historicalRecord = {
id: historicalDocumentId,
kind: "benchmark.history",
@ -176,7 +187,7 @@ export async function seedStorageBenchmark(
],
BACKGROUND_CONTEXT,
);
let ancientAt = 0;
let ancientAt: Seq | undefined;
for (let count = 1; count <= HISTORY_SEGMENT_LENGTH; count++) {
ancientAt = await storage.commit(
[
@ -199,6 +210,7 @@ export async function seedStorageBenchmark(
],
BACKGROUND_CONTEXT,
);
if (ancientAt === undefined) throw new Error("Benchmark history seed produced no commits");
let recentAt = ancientAt;
for (let count = HISTORY_SEGMENT_LENGTH + 1; count <= HISTORY_SEGMENT_LENGTH * 2; count++) {
recentAt = await storage.commit(
@ -213,11 +225,12 @@ export async function seedStorageBenchmark(
);
}
if (firstEntryId === undefined) throw new Error("Benchmark scale must create entries");
let parentConversationId = ROOT_CONVERSATION_ID;
let parentAt = firstEntryId;
let deepestConversationId = ROOT_CONVERSATION_ID;
for (let depth = 0; depth < FORK_DEPTH; depth++) {
const conversationId = await storage.mintId();
const conversationId = await storage.mintId<ConversationId>();
await storage.commit(
[
{
@ -227,7 +240,7 @@ export async function seedStorageBenchmark(
],
BACKGROUND_CONTEXT,
);
const ids = await Promise.all(Array.from({ length: ENTRIES_PER_FORK }, () => storage.mintId()));
const ids = await Promise.all(Array.from({ length: ENTRIES_PER_FORK }, () => storage.mintId<EntryId>()));
await storage.commit(
ids.map((id, index) => ({
type: "entry" as const,
@ -245,6 +258,7 @@ export async function seedStorageBenchmark(
deepestConversationId = conversationId;
}
if (exactDocumentId === undefined) throw new Error("Benchmark scale must create documents");
return {
firstEntryId,
filteredTaskCount: Math.min(50, Math.ceil(scale.taskCount / 60)),
@ -252,7 +266,7 @@ export async function seedStorageBenchmark(
exactDocumentKey: `key-${scale.documentCount - 1}`,
replayDocumentIds: Object.fromEntries(replayEntries.map(({ tail, id }) => [tail, id])) as Record<
(typeof REPLAY_TAILS)[number],
Id
DocumentId
>,
historicalDocumentId,
ancientAt,
@ -382,7 +396,7 @@ export async function seedStorageWriteBenchmark(storage: Storage): Promise<void>
Array.from({ length: 100 }, async (_, index) => ({
type: "entry" as const,
value: {
id: await storage.mintId(),
id: await storage.mintId<EntryId>(),
conversationId: ROOT_CONVERSATION_ID,
kind: "benchmark.baseline",
data: { index },
@ -398,7 +412,7 @@ export const STORAGE_WRITE_BENCHMARKS: readonly StorageWriteBenchmark[] = [
name: "commit one entry",
expected: 1,
async run(storage) {
const id = await storage.mintId();
const id = await storage.mintId<EntryId>();
await storage.commit(
[
{
@ -426,7 +440,7 @@ export const STORAGE_WRITE_BENCHMARKS: readonly StorageWriteBenchmark[] = [
async (_, index): Promise<StorageWrite> => ({
type: "entry",
value: {
id: await storage.mintId(),
id: await storage.mintId<EntryId>(),
conversationId: ROOT_CONVERSATION_ID,
kind: "benchmark.write",
data: { index, text: "x".repeat(128) },
@ -442,10 +456,10 @@ export const STORAGE_WRITE_BENCHMARKS: readonly StorageWriteBenchmark[] = [
name: "commit mixed entry/task/submission/document",
expected: 4,
async run(storage) {
const entryId = await storage.mintId();
const taskId = await storage.mintId();
const submissionId = await storage.mintId();
const documentId = await storage.mintId();
const entryId = await storage.mintId<EntryId>();
const taskId = await storage.mintId<TaskId<JsonValue>>();
const submissionId = await storage.mintId<SubmissionId>();
const documentId = await storage.mintId<DocumentId>();
const writes: readonly StorageWrite[] = [
{
type: "entry",

View file

@ -1,15 +1,20 @@
import type { JsonValue } from "@earendil-works/chord";
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
import type { Op } from "@earendil-works/chord/delta";
import { idFromNumber } from "../ids.ts";
import {
type ConversationId,
type DocumentCreate,
type DocumentId,
type EntryId,
type EntryRecord,
type Id,
type JsonObject,
ROOT_CONVERSATION_ID,
type Storage,
type StorageWrite,
type SubmissionId,
type SubmissionRecord,
type TaskId,
type TaskRecord,
} from "../types.ts";
import type { StorageConformanceAssertions, StorageConformanceCase, StorageConformanceOptions } from "./types.ts";
@ -17,12 +22,12 @@ import type { StorageConformanceAssertions, StorageConformanceCase, StorageConfo
const context = BACKGROUND_CONTEXT;
type StoredTask = TaskRecord<JsonValue, JsonValue, JsonValue>;
async function createRoot(storage: Storage): Promise<Id> {
async function createRoot(storage: Storage): Promise<ConversationId> {
await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context);
return ROOT_CONVERSATION_ID;
}
function pendingTask(id: Id, conversationId: Id, phase = "ready") {
function pendingTask(id: TaskId<JsonValue>, conversationId: ConversationId, phase = "ready") {
return {
id,
conversationId,
@ -36,7 +41,12 @@ function pendingTask(id: Id, conversationId: Id, phase = "ready") {
} satisfies StoredTask;
}
function entry(id: Id, conversationId: Id, kind = "message", extra: Partial<EntryRecord> = {}): EntryRecord {
function entry(
id: EntryId,
conversationId: ConversationId,
kind = "message",
extra: Partial<EntryRecord> = {},
): EntryRecord {
return { id, conversationId, kind, ...extra };
}
@ -81,7 +91,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re
const expect = assertionFacade(options.assertions);
return [
createCase(options, "reserves ID 1 for the immutable root conversation", async (storage) => {
expect(await storage.mintId()).toBe(2);
expect(await storage.mintId<ConversationId>()).toBe(2);
await expect(createRoot(storage)).resolves.toBe(ROOT_CONVERSATION_ID);
expect(await storage.conversation(ROOT_CONVERSATION_ID, context)).toEqual({ id: ROOT_CONVERSATION_ID });
await expect(
@ -94,9 +104,9 @@ export function createStorageConformance(options: StorageConformanceOptions): re
"commits mixed table writes atomically and rolls all of them back on failure",
async (storage) => {
const rootId = await createRoot(storage);
const entryId = await storage.mintId();
const taskId = await storage.mintId();
const submissionId = await storage.mintId();
const entryId = await storage.mintId<EntryId>();
const taskId = await storage.mintId<TaskId<JsonValue>>();
const submissionId = await storage.mintId<SubmissionId>();
const task = pendingTask(taskId, rootId);
const input: SubmissionRecord = {
id: submissionId,
@ -122,7 +132,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re
expect(await storage.task(taskId, context)).toEqual(task);
expect(await storage.submission(submissionId, context)).toEqual(input);
const transientEntryId = await storage.mintId();
const transientEntryId = await storage.mintId<EntryId>();
const runningTask: StoredTask = {
...task,
state: { status: "running", checkpoint: { phase: "effect" } },
@ -144,7 +154,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re
expect(await storage.submission(submissionId, context)).toEqual(input);
expect(await storage.entry(transientEntryId, context)).toBeUndefined();
const afterRollbackSeq = await storage.commit(
[{ type: "entry", value: entry(await storage.mintId(), rootId, "after-rollback") }],
[{ type: "entry", value: entry(await storage.mintId<EntryId>(), rootId, "after-rollback") }],
context,
);
expect(afterRollbackSeq).toBeGreaterThan(initialSeq);
@ -153,9 +163,9 @@ export function createStorageConformance(options: StorageConformanceOptions): re
createCase(options, "detaches retained writes and every returned record", async (storage) => {
const rootId = await createRoot(storage);
const entryId = await storage.mintId();
const taskId = await storage.mintId();
const submissionId = await storage.mintId();
const entryId = await storage.mintId<EntryId>();
const taskId = await storage.mintId<TaskId<JsonValue>>();
const submissionId = await storage.mintId<SubmissionId>();
const entryData = { nested: [1, 2] };
const checkpoint = { phase: "ready", nested: { count: 1 } };
const detail = { codes: ["initial"] };
@ -210,7 +220,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re
createCase(options, "detaches prototype-like JSON keys without changing object prototypes", async (storage) => {
const rootId = await createRoot(storage);
const entryId = await storage.mintId();
const entryId = await storage.mintId<EntryId>();
const data = JSON.parse(
'{"__proto__":{"polluted":false},"constructor":{"label":"stored"},"toString":"value"}',
) as Record<string, JsonValue>;
@ -237,9 +247,14 @@ export function createStorageConformance(options: StorageConformanceOptions): re
const rootId = await createRoot(storage);
await storage.commit(
[
{ type: "entry", value: entry(30, rootId) },
{ type: "entry", value: entry(10, rootId) },
{ type: "entry", value: entry(20, rootId, "marker", { head: 10 }) },
{ type: "entry", value: entry(idFromNumber<EntryId>(30), rootId) },
{ type: "entry", value: entry(idFromNumber<EntryId>(10), rootId) },
{
type: "entry",
value: entry(idFromNumber<EntryId>(20), rootId, "marker", {
head: idFromNumber<EntryId>(10),
}),
},
],
context,
);
@ -252,9 +267,9 @@ export function createStorageConformance(options: StorageConformanceOptions): re
createCase(options, "continues an entry cursor below its last item after a newer commit", async (storage) => {
const rootId = await createRoot(storage);
const oldestId = await storage.mintId();
const middleId = await storage.mintId();
const newestId = await storage.mintId();
const oldestId = await storage.mintId<EntryId>();
const middleId = await storage.mintId<EntryId>();
const newestId = await storage.mintId<EntryId>();
await storage.commit(
[
{ type: "entry", value: entry(oldestId, rootId) },
@ -266,7 +281,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re
const first = await storage.scanEntries({ conversationId: rootId }, 2, undefined, context);
expect(first.items.map(({ id }) => id)).toEqual([newestId, middleId]);
const appendedId = await storage.mintId();
const appendedId = await storage.mintId<EntryId>();
await storage.commit([{ type: "entry", value: entry(appendedId, rootId) }], context);
const second = await storage.scanEntries({ conversationId: rootId }, 2, first.next, context);
expect(second.items.map(({ id }) => id)).toEqual([oldestId]);
@ -275,8 +290,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re
createCase(options, "paginates conversations by opaque cursor in ascending ID order", async (storage) => {
const rootId = await createRoot(storage);
const secondId = await storage.mintId();
const thirdId = await storage.mintId();
const secondId = await storage.mintId<ConversationId>();
const thirdId = await storage.mintId<ConversationId>();
await storage.commit(
[
{ type: "conversation", value: { id: thirdId } },
@ -285,20 +300,70 @@ export function createStorageConformance(options: StorageConformanceOptions): re
context,
);
const first = await storage.scanConversations(2, undefined, context);
const first = await storage.scanConversations({}, 2, undefined, context);
expect(first.items.map(({ id }) => id)).toEqual([rootId, secondId]);
expect(first.next).toBeDefined();
const roundTrippedCursor = JSON.parse(JSON.stringify(first.next)) as NonNullable<typeof first.next>;
const second = await storage.scanConversations(2, roundTrippedCursor, context);
const second = await storage.scanConversations({}, 2, roundTrippedCursor, context);
expect(second.items.map(({ id }) => id)).toEqual([thirdId]);
expect(second.next).toBeUndefined();
}),
createCase(options, "filters and pages conversations by durable owner edges", async (storage) => {
const rootId = await createRoot(storage);
const otherOwnerId = await storage.mintId<ConversationId>();
const firstTaskId = await storage.mintId<TaskId<JsonValue>>();
const secondTaskId = await storage.mintId<TaskId<JsonValue>>();
const firstId = await storage.mintId<ConversationId>();
const secondId = await storage.mintId<ConversationId>();
const thirdId = await storage.mintId<ConversationId>();
await storage.commit(
[
{ type: "conversation", value: { id: otherOwnerId } },
{
type: "conversation",
value: { id: firstId, owner: { conversationId: rootId, taskId: firstTaskId } },
},
{
type: "conversation",
value: { id: secondId, owner: { conversationId: rootId, taskId: secondTaskId } },
},
{
type: "conversation",
value: { id: thirdId, owner: { conversationId: otherOwnerId, taskId: firstTaskId } },
},
],
context,
);
const first = await storage.scanConversations({ ownerConversationId: rootId }, 1, undefined, context);
expect(first.items.map(({ id }) => id)).toEqual([firstId]);
expect(first.next).toBeDefined();
const second = await storage.scanConversations({ ownerConversationId: rootId }, 1, first.next, context);
expect(second.items.map(({ id }) => id)).toEqual([secondId]);
expect(second.next).toBeUndefined();
expect(
(await storage.scanConversations({ ownerTaskId: firstTaskId }, 10, undefined, context)).items.map(
({ id }) => id,
),
).toEqual([firstId, thirdId]);
expect(
(
await storage.scanConversations(
{ ownerConversationId: rootId, ownerTaskId: firstTaskId },
10,
undefined,
context,
)
).items.map(({ id }) => id),
).toEqual([firstId]);
}),
createCase(options, "scans deep fork history newest-first through every ancestor cap", async (storage) => {
const rootId = await createRoot(storage);
const rootFirst = await storage.mintId();
const rootForkPoint = await storage.mintId();
const rootExcludedSameCommit = await storage.mintId();
const rootFirst = await storage.mintId<EntryId>();
const rootForkPoint = await storage.mintId<EntryId>();
const rootExcludedSameCommit = await storage.mintId<EntryId>();
const rootEntriesSeq = await storage.commit(
[
{ type: "entry", value: entry(rootFirst, rootId) },
@ -310,7 +375,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re
],
context,
);
const childId = await storage.mintId();
const childId = await storage.mintId<ConversationId>();
await storage.commit(
[
{
@ -320,8 +385,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re
],
context,
);
const childForkPoint = await storage.mintId();
const childExcluded = await storage.mintId();
const childForkPoint = await storage.mintId<EntryId>();
const childExcluded = await storage.mintId<EntryId>();
await storage.commit(
[
{ type: "entry", value: entry(childForkPoint, childId, "note") },
@ -329,9 +394,9 @@ export function createStorageConformance(options: StorageConformanceOptions): re
],
context,
);
const rootExcludedLater = await storage.mintId();
const rootExcludedLater = await storage.mintId<EntryId>();
await storage.commit([{ type: "entry", value: entry(rootExcludedLater, rootId) }], context);
const grandchildId = await storage.mintId();
const grandchildId = await storage.mintId<ConversationId>();
await storage.commit(
[
{
@ -341,8 +406,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re
],
context,
);
const grandchildHead = await storage.mintId();
const grandchildTail = await storage.mintId();
const grandchildHead = await storage.mintId<EntryId>();
const grandchildTail = await storage.mintId<EntryId>();
const grandchildEntriesSeq = await storage.commit(
[
{
@ -353,7 +418,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re
],
context,
);
const childExcludedLater = await storage.mintId();
const childExcludedLater = await storage.mintId<EntryId>();
await storage.commit([{ type: "entry", value: entry(childExcludedLater, childId) }], context);
const first = await storage.scanEntries({ conversationId: grandchildId }, 2, undefined, context);
@ -411,7 +476,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re
expect((await storage.entry(rootForkPoint, context))?.commitSeq).toBe(rootEntriesSeq);
expect((await storage.entry(grandchildHead, context))?.commitSeq).toBe(grandchildEntriesSeq);
expect((await storage.entry(grandchildTail, context))?.commitSeq).toBe(grandchildEntriesSeq);
expect(await storage.entry(999_999, context)).toBeUndefined();
expect(await storage.entry(idFromNumber<EntryId>(999_999), context)).toBeUndefined();
expect(await storage.entry(grandchildId, rootFirst, context)).toEqual({
entry: entry(rootFirst, rootId),
@ -424,18 +489,20 @@ export function createStorageConformance(options: StorageConformanceOptions): re
expect(await storage.entry(grandchildId, childExcluded, context)).toBeUndefined();
expect(await storage.entry(grandchildId, childExcludedLater, context)).toBeUndefined();
expect(await storage.entry(rootId, grandchildHead, context)).toBeUndefined();
expect(await storage.entry(grandchildId, 999_999, context)).toBeUndefined();
await expect(storage.entry(999_999, rootFirst, context)).rejects.toThrow("Unknown conversation");
await expect(storage.scanEntries({ conversationId: 999_999 }, 10, undefined, context)).rejects.toThrow(
expect(await storage.entry(grandchildId, idFromNumber<EntryId>(999_999), context)).toBeUndefined();
await expect(storage.entry(idFromNumber<ConversationId>(999_999), rootFirst, context)).rejects.toThrow(
"Unknown conversation",
);
await expect(
storage.scanEntries({ conversationId: idFromNumber<ConversationId>(999_999) }, 10, undefined, context),
).rejects.toThrow("Unknown conversation");
}),
createCase(options, "replaces complete task records and pages filtered task scans", async (storage) => {
const rootId = await createRoot(storage);
const firstId = await storage.mintId();
const secondId = await storage.mintId();
const thirdId = await storage.mintId();
const firstId = await storage.mintId<TaskId<JsonValue>>();
const secondId = await storage.mintId<TaskId<JsonValue>>();
const thirdId = await storage.mintId<TaskId<JsonValue>>();
const first = { ...pendingTask(firstId, rootId), memos: { winner: "first" } } satisfies StoredTask;
const second = { ...pendingTask(secondId, rootId), background: true } satisfies StoredTask;
const third = { ...pendingTask(thirdId, rootId), abortRequested: true } satisfies StoredTask;
@ -488,11 +555,11 @@ export function createStorageConformance(options: StorageConformanceOptions): re
"indexes request IDs per conversation and replaces complete submission records",
async (storage) => {
const rootId = await createRoot(storage);
const secondConversationId = await storage.mintId();
const secondConversationId = await storage.mintId<ConversationId>();
await storage.commit([{ type: "conversation", value: { id: secondConversationId } }], context);
const firstId = await storage.mintId();
const secondId = await storage.mintId();
const otherConversationId = await storage.mintId();
const firstId = await storage.mintId<SubmissionId>();
const secondId = await storage.mintId<SubmissionId>();
const otherConversationId = await storage.mintId<SubmissionId>();
const first: SubmissionRecord = {
id: firstId,
conversationId: rootId,
@ -525,7 +592,11 @@ export function createStorageConformance(options: StorageConformanceOptions): re
expect(await storage.submissionByRequest(rootId, "same", context)).toEqual(first);
expect(await storage.submissionByRequest(secondConversationId, "same", context)).toEqual(otherConversation);
const placedSecond: SubmissionRecord = { ...second, status: "placed", entry: await storage.mintId() };
const placedSecond: SubmissionRecord = {
...second,
status: "placed",
entry: await storage.mintId<EntryId>(),
};
await storage.commit([{ type: "submission", value: placedSecond }], context);
expect(await storage.submission(secondId, context)).toEqual(placedSecond);
expect(await storage.submissionByRequest(rootId, "other", context)).toEqual(placedSecond);
@ -534,8 +605,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re
createCase(options, "stores passive write submissions without input-only lifecycle states", async (storage) => {
const rootId = await createRoot(storage);
const doneId = await storage.mintId();
const failedId = await storage.mintId();
const doneId = await storage.mintId<SubmissionId>();
const failedId = await storage.mintId<SubmissionId>();
const queuedDone: SubmissionRecord = {
id: doneId,
conversationId: rootId,
@ -558,7 +629,11 @@ export function createStorageConformance(options: StorageConformanceOptions): re
context,
);
const done: SubmissionRecord = { ...queuedDone, status: "done", entry: await storage.mintId() };
const done: SubmissionRecord = {
...queuedDone,
status: "done",
entry: await storage.mintId<EntryId>(),
};
const unanswered: SubmissionRecord = {
...queuedFailed,
status: "unanswered",
@ -580,7 +655,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re
createCase(options, "reconstructs rewindable documents and preserves half-open incarnations", async (storage) => {
const rootId = await createRoot(storage);
const firstId = await storage.mintId();
const firstId = await storage.mintId<DocumentId>();
const firstRecord = {
id: firstId,
kind: "conversation.notes",
@ -660,7 +735,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re
nested: { count: 4 },
});
const secondId = await storage.mintId();
const secondId = await storage.mintId<DocumentId>();
const retiredAt = await storage.commit(
[
{
@ -699,7 +774,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re
createCase(options, "streams long document tails across root replacement deltas", async (storage) => {
const rootId = await createRoot(storage);
const id = await storage.mintId();
const id = await storage.mintId<DocumentId>();
const record = {
id,
kind: "conversation.long-tail",
@ -790,12 +865,212 @@ export function createStorageConformance(options: StorageConformanceOptions): re
expect((await storage.document(id, "current", context))?.value).toEqual(current);
}),
createCase(
options,
"copies stored document bases independently and rejects ambiguous sources",
async (storage) => {
const rootId = await createRoot(storage);
const childId = await storage.mintId<ConversationId>();
const secondChildId = await storage.mintId<ConversationId>();
await storage.commit(
[
{ type: "conversation", value: { id: childId } },
{ type: "conversation", value: { id: secondChildId } },
],
context,
);
const sourceId = await storage.mintId<DocumentId>();
const sourceRecord = {
id: sourceId,
kind: "copy.source",
scope: { kind: "conversation", conversationId: rootId },
history: "rewindable",
fork: "asOf",
} satisfies DocumentCreate;
const createdAt = await storage.commit(
[
{
type: "document.create",
record: sourceRecord,
content: { kind: "base", version: 2, value: { count: 1, rows: [{ value: "base" }] } },
},
],
context,
);
await storage.commit(
[
{
type: "document.change",
id: sourceId,
content: {
kind: "delta",
version: 2,
ops: [
["s", ["count"], 2],
["p", ["rows"], 1, 0, [{ value: "current" }]],
],
},
},
],
context,
);
const historicalCopyId = await storage.mintId<DocumentId>();
const currentCopyId = await storage.mintId<DocumentId>();
const retiredCopyId = await storage.mintId<DocumentId>();
const childRecord = (id: DocumentId, conversationId: ConversationId): DocumentCreate => ({
id,
kind: sourceRecord.kind,
scope: { kind: "conversation", conversationId },
history: "rewindable",
fork: "asOf",
});
await storage.commit(
[
{
type: "document.copy",
record: childRecord(historicalCopyId, childId),
source: { id: sourceId, at: createdAt },
},
{
type: "document.copy",
record: childRecord(currentCopyId, secondChildId),
source: { id: sourceId, at: "current" },
},
{
type: "document.copy",
record: childRecord(retiredCopyId, rootId),
source: { id: sourceId, at: "current" },
},
{ type: "document.retire", id: retiredCopyId },
],
context,
);
expect(await storage.document(historicalCopyId, "current", context)).toMatchObject({
version: 2,
value: { count: 1, rows: [{ value: "base" }] },
});
expect(await storage.document(currentCopyId, "current", context)).toMatchObject({
version: 2,
value: { count: 2, rows: [{ value: "base" }, { value: "current" }] },
});
expect(await storage.document(retiredCopyId, "current", context)).toBeUndefined();
await storage.commit(
[
{
type: "document.change",
id: sourceId,
content: { kind: "base", version: 2, value: { count: 99, rows: [] } },
},
{ type: "document.retire", id: sourceId },
],
context,
);
expect((await storage.document(currentCopyId, "current", context))?.value).toEqual({
count: 2,
rows: [{ value: "base" }, { value: "current" }],
});
const latestSourceId = await storage.mintId<DocumentId>();
const latestCopyId = await storage.mintId<DocumentId>();
const latestSource = {
id: latestSourceId,
kind: "copy.latest",
scope: { kind: "conversation", conversationId: rootId },
history: "latest",
fork: "current",
} satisfies DocumentCreate;
await storage.commit(
[
{
type: "document.create",
record: latestSource,
content: { kind: "base", version: 4, value: { retained: "copy" } },
},
],
context,
);
await storage.commit(
[
{
type: "document.copy",
record: {
...latestSource,
id: latestCopyId,
scope: { kind: "conversation", conversationId: childId },
},
source: { id: latestSourceId, at: "current" },
},
],
context,
);
await storage.commit(
[
{
type: "document.change",
id: latestSourceId,
content: { kind: "base", version: 4, value: { retained: "source-only" } },
},
{ type: "document.retire", id: latestSourceId },
],
context,
);
expect(await storage.document(latestCopyId, "current", context)).toMatchObject({
version: 4,
value: { retained: "copy" },
});
const conflictId = await storage.mintId<DocumentId>();
let conflictError: unknown;
try {
await storage.commit(
[
{
type: "document.copy",
record: childRecord(conflictId, childId),
source: { id: currentCopyId, at: "current" },
},
{ type: "document.retire", id: currentCopyId },
],
context,
);
} catch (error) {
conflictError = error;
}
expect((conflictError as Error | undefined)?.name).toBe("StorageRejected");
expect(await storage.document(conflictId, "current", context)).toBeUndefined();
expect((await storage.document(currentCopyId, "current", context))?.value).toEqual({
count: 2,
rows: [{ value: "base" }, { value: "current" }],
});
const mismatchId = await storage.mintId<DocumentId>();
let mismatchError: unknown;
try {
await storage.commit(
[
{
type: "document.copy",
record: { ...childRecord(mismatchId, childId), kind: "copy.mismatch" },
source: { id: currentCopyId, at: "current" },
},
],
context,
);
} catch (error) {
mismatchError = error;
}
expect((mismatchError as Error | undefined)?.name).toBe("StorageRejected");
expect(await storage.document(mismatchId, "current", context)).toBeUndefined();
},
),
createCase(
options,
"uses bases for version transitions and rejects historical reads of current-only documents",
async (storage) => {
await createRoot(storage);
const id = await storage.mintId();
const id = await storage.mintId<DocumentId>();
const record = {
id,
kind: "session.settings",
@ -832,13 +1107,13 @@ export function createStorageConformance(options: StorageConformanceOptions): re
createCase(options, "indexes logical addresses and exact-scope scans independently", async (storage) => {
const rootId = await createRoot(storage);
const firstId = await storage.mintId();
const secondId = await storage.mintId();
const conversationId = await storage.mintId();
const taskId = await storage.mintId();
const taskSingletonId = await storage.mintId();
const taskFamilyId = await storage.mintId();
const taskOtherKindId = await storage.mintId();
const firstId = await storage.mintId<DocumentId>();
const secondId = await storage.mintId<DocumentId>();
const conversationId = await storage.mintId<DocumentId>();
const taskId = await storage.mintId<TaskId<JsonValue>>();
const taskSingletonId = await storage.mintId<DocumentId>();
const taskFamilyId = await storage.mintId<DocumentId>();
const taskOtherKindId = await storage.mintId<DocumentId>();
const createdAt = await storage.commit(
[
{ type: "task", value: pendingTask(taskId, rootId) },
@ -974,8 +1249,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re
"keeps document lifecycle failures atomic and gives create-plus-retire an empty lifetime",
async (storage) => {
const rootId = await createRoot(storage);
const firstId = await storage.mintId();
const secondId = await storage.mintId();
const firstId = await storage.mintId<DocumentId>();
const secondId = await storage.mintId<DocumentId>();
const record = {
id: firstId,
kind: "singleton",
@ -1001,7 +1276,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re
expect((await storage.document(firstId, "current", context))?.value).toEqual({ value: 1 });
expect(await storage.document(secondId, "current", context)).toBeUndefined();
const emptyId = await storage.mintId();
const emptyId = await storage.mintId<DocumentId>();
const emptyAt = await storage.commit(
[
{
@ -1041,9 +1316,9 @@ export function createStorageConformance(options: StorageConformanceOptions): re
"rolls back record tables and secondary indexes when a document command fails",
async (storage) => {
const rootId = await createRoot(storage);
const taskId = await storage.mintId();
const submissionId = await storage.mintId();
const documentId = await storage.mintId();
const taskId = await storage.mintId<TaskId<JsonValue>>();
const submissionId = await storage.mintId<SubmissionId>();
const documentId = await storage.mintId<DocumentId>();
const task = pendingTask(taskId, rootId);
const submission: SubmissionRecord = {
id: submissionId,
@ -1066,8 +1341,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re
context,
);
const entryId = await storage.mintId();
const conflictingDocumentId = await storage.mintId();
const entryId = await storage.mintId<EntryId>();
const conflictingDocumentId = await storage.mintId<DocumentId>();
await expect(
storage.commit(
[
@ -1116,14 +1391,14 @@ export function createStorageConformance(options: StorageConformanceOptions): re
const rootId = await createRoot(storage);
const first = "\ud800";
const second = "\ud801";
const firstTaskId = await storage.mintId();
const secondTaskId = await storage.mintId();
const firstSubmissionId = await storage.mintId();
const secondSubmissionId = await storage.mintId();
const firstKindDocumentId = await storage.mintId();
const secondKindDocumentId = await storage.mintId();
const firstKeyDocumentId = await storage.mintId();
const secondKeyDocumentId = await storage.mintId();
const firstTaskId = await storage.mintId<TaskId<JsonValue>>();
const secondTaskId = await storage.mintId<TaskId<JsonValue>>();
const firstSubmissionId = await storage.mintId<SubmissionId>();
const secondSubmissionId = await storage.mintId<SubmissionId>();
const firstKindDocumentId = await storage.mintId<DocumentId>();
const secondKindDocumentId = await storage.mintId<DocumentId>();
const firstKeyDocumentId = await storage.mintId<DocumentId>();
const secondKeyDocumentId = await storage.mintId<DocumentId>();
await storage.commit(
[
{ type: "task", value: { ...pendingTask(firstTaskId, rootId), kind: first } },
@ -1216,16 +1491,22 @@ export function createStorageConformance(options: StorageConformanceOptions): re
createCase(options, "keeps one global record ID namespace and rejects exhausted ID minting", async (storage) => {
const rootId = await createRoot(storage);
const explicitEntryId = 100;
const explicitEntryId = idFromNumber<EntryId>(100);
await storage.commit([{ type: "entry", value: entry(explicitEntryId, rootId) }], context);
expect(await storage.mintId()).toBe(101);
expect(await storage.mintId<EntryId>()).toBe(101);
await expect(
storage.commit([{ type: "task", value: pendingTask(explicitEntryId, rootId) }], context),
storage.commit(
[{ type: "task", value: pendingTask(idFromNumber<TaskId<JsonValue>>(explicitEntryId), rootId) }],
context,
),
).rejects.toThrow(`ID ${explicitEntryId} already belongs to entry`);
await storage.commit([{ type: "entry", value: entry(Number.MAX_SAFE_INTEGER, rootId, "last-id") }], context);
await expect(storage.mintId()).rejects.toThrow("ID space is exhausted");
await expect(storage.mintId()).rejects.toThrow("ID space is exhausted");
await storage.commit(
[{ type: "entry", value: entry(idFromNumber<EntryId>(Number.MAX_SAFE_INTEGER), rootId, "last-id") }],
context,
);
await expect(storage.mintId<EntryId>()).rejects.toThrow("ID space is exhausted");
await expect(storage.mintId<EntryId>()).rejects.toThrow("ID space is exhausted");
}),
createCase(options, "rejects every operation after close", async (storage) => {
@ -1233,7 +1514,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re
await storage.close(context);
await expect(storage.conversation(ROOT_CONVERSATION_ID, context)).rejects.toThrow("closed");
await expect(storage.commit([] satisfies StorageWrite[], context)).rejects.toThrow("closed");
await expect(storage.mintId()).rejects.toThrow("closed");
await expect(storage.mintId<ConversationId>()).rejects.toThrow("closed");
}),
];
}

View file

@ -5,14 +5,29 @@ import type { Message } from "@earendil-works/pi-ai";
/** JSON object used as the root of every durable document. */
export type JsonObject = { [key: string]: JsonValue };
/** Session-global identifier shared by every durable record table. */
export type Id = number;
declare const idBrand: unique symbol;
/** Erased nominal number identifying one durable record kind. */
export type Id<Kind extends string, Type = unknown> = number & {
readonly [idBrand]: {
readonly kind: Kind;
readonly type: Type;
};
};
export type ConversationId = Id<"conversation">;
export type EntryId = Id<"entry">;
export type TaskId<Result = unknown> = Id<"task", Result>;
export type SubmissionId = Id<"submission">;
export type DocumentId = Id<"document">;
declare const seqBrand: unique symbol;
/** Strictly increasing sequence assigned to one atomic storage commit; gaps are permitted. */
export type Seq = number;
export type Seq = number & { readonly [seqBrand]: "sequence" };
/** The root conversation always uses this reserved ID. */
export const ROOT_CONVERSATION_ID: Id = 1;
export const ROOT_CONVERSATION_ID = 1 as ConversationId;
/** Conversation document that retains only its current state. */
export type LatestConversationSemantics = {
@ -122,38 +137,38 @@ export interface Task<I, S extends { phase: string }, R, H extends object> {
readonly definition: TaskDefinition<I, S, R, H>;
}
/** Task ID carrying its result type for typed waits. */
export type TaskRef<R> = { readonly id: Id; readonly [taskResultType]?: R };
/** Creation options for a durable task. */
export type TaskOptions = {
/** Owning conversation; required for Session commits that are not bound to a conversation. */
readonly conversationId?: Id;
readonly conversationId?: ConversationId;
/** Tasks that must be terminal before ordinary execution may begin. */
readonly after?: readonly Id[];
readonly after?: readonly TaskId[];
/** Excluded from ordinary idle waits and conversation aborts. */
readonly background?: boolean;
};
/** Ownership selected explicitly whenever a conversation is created. */
export type ConversationOwnership = { readonly kind: "ownerless" } | { readonly kind: "task"; readonly taskId: TaskId };
/** Immutable identity, history ancestry, and task ownership of a transcript scope. */
export type ConversationRecord = {
readonly id: Id;
readonly id: ConversationId;
/** Fork source and inclusive parent entry through which history is inherited. */
readonly parent?: {
readonly conversationId: Id;
readonly at: Id;
readonly conversationId: ConversationId;
readonly at: EntryId;
};
/** Creator edge used for authorization, subtree abort, and subtree idle waits. */
/** Creator edge used for attribution, subtree abort, and subtree idle waits. */
readonly owner?: {
readonly conversationId: Id;
readonly taskId: Id;
readonly conversationId: ConversationId;
readonly taskId: TaskId;
};
};
/** An immutable override of one visible entry's contribution to model context. */
export type ContextEdit = {
/** Entry whose model messages are omitted or replaced. */
readonly target: Id;
readonly target: EntryId;
} & (
| {
readonly action: "omit";
@ -168,8 +183,8 @@ export type ContextEdit = {
/** Immutable transcript event with separate model-facing and application-facing payloads. */
export type EntryRecord = {
readonly id: Id;
readonly conversationId: Id;
readonly id: EntryId;
readonly conversationId: ConversationId;
/** Application-defined entry discriminator. */
readonly kind: string;
/** Messages contributed to model context; absent for display or bookkeeping entries. */
@ -177,23 +192,23 @@ export type EntryRecord = {
/** JSON payload consumed by views, extensions, or bookkeeping logic. */
readonly data?: JsonValue;
/** First entry in the active context selected by this entry. */
readonly head?: Id;
readonly head?: EntryId;
/** Context-only overrides of earlier visible entries. */
readonly edits?: readonly ContextEdit[];
/** Task that appended this entry, when it was produced by durable work. */
readonly byTaskId?: Id;
readonly byTaskId?: TaskId;
};
/** Entry content supplied before the Session assigns identity and task attribution. */
export type EntryDraft = Omit<EntryRecord, "id" | "conversationId" | "byTaskId" | "head"> & {
/** `"self"` starts active context at the newly assigned entry ID. */
readonly head?: Id | "self";
readonly head?: EntryId | "self";
};
/** Identity fields shared by every durable submission state. */
type SubmissionRecordBase = {
readonly id: Id;
readonly conversationId: Id;
readonly id: SubmissionId;
readonly conversationId: ConversationId;
/** Host-provided deduplication key, scoped to the conversation. */
readonly requestId?: string;
};
@ -214,7 +229,7 @@ export type SubmissionRecord =
| {
/** Added to the transcript and owned by an active turn. */
readonly status: "placed";
readonly entry: Id;
readonly entry: EntryId;
readonly answer?: never;
readonly reason?: never;
readonly detail?: never;
@ -222,15 +237,15 @@ export type SubmissionRecord =
| {
/** Successfully answered user input. */
readonly status: "done";
readonly entry: Id;
readonly answer: Id;
readonly entry: EntryId;
readonly answer: EntryId;
readonly reason?: never;
readonly detail?: never;
}
| {
/** Terminal input that can no longer receive an answer. */
readonly status: "unanswered";
readonly entry?: Id;
readonly entry?: EntryId;
readonly answer?: never;
readonly reason: string;
readonly detail?: JsonValue;
@ -250,7 +265,7 @@ export type SubmissionRecord =
| {
/** Successfully appended passive entry. */
readonly status: "done";
readonly entry: Id;
readonly entry: EntryId;
readonly answer?: never;
readonly reason?: never;
readonly detail?: never;
@ -340,9 +355,9 @@ export type TaskState<S, R> =
};
/** Identity, definition, and scheduling fields shared by every task state. */
type TaskRecordBase<I> = {
readonly id: Id;
readonly conversationId: Id;
type TaskRecordBase<I, R> = {
readonly id: TaskId<R>;
readonly conversationId: ConversationId;
/** Registered task definition name. */
readonly kind: string;
/** Definition version used to migrate live input and checkpoints. */
@ -350,7 +365,7 @@ type TaskRecordBase<I> = {
/** Original task input retained while the task is live or terminal. */
readonly input: I;
/** Tasks that must be terminal before ordinary execution may begin. */
readonly after: readonly Id[];
readonly after: readonly TaskId[];
/** Whether this task is excluded from ordinary idle waits and conversation aborts. */
readonly background: boolean;
/** Durable abort mark checked before run-mode progress is committed. */
@ -358,7 +373,7 @@ type TaskRecordBase<I> = {
};
/** Complete replacement record for one durable task state machine. */
export type TaskRecord<I, S, R> = TaskRecordBase<I> &
export type TaskRecord<I, S, R> = TaskRecordBase<I, R> &
(
| {
readonly state: Extract<TaskState<S, R>, { readonly status: "pending" | "running" }>;
@ -374,7 +389,7 @@ export type TaskRecord<I, S, R> = TaskRecordBase<I> &
/** Persisted lifecycle record for one create-to-retire document incarnation. */
export type DocumentRecord = {
/** Unique incarnation ID; never reused when the same logical document is recreated. */
readonly id: Id;
readonly id: DocumentId;
/** Stable document definition kind. */
readonly kind: string;
/** Family member key; absent for singleton documents. */
@ -389,7 +404,7 @@ export type DocumentRecord = {
readonly history?: never;
readonly fork?: never;
}
| ({ readonly scope: { readonly kind: "conversation"; readonly conversationId: Id } } & (
| ({ readonly scope: { readonly kind: "conversation"; readonly conversationId: ConversationId } } & (
| {
/** Retain only current state. */
readonly history: "latest";
@ -404,7 +419,7 @@ export type DocumentRecord = {
}
))
| {
readonly scope: { readonly kind: "task"; readonly taskId: Id };
readonly scope: { readonly kind: "task"; readonly taskId: TaskId };
readonly history?: never;
readonly fork?: never;
}
@ -426,18 +441,24 @@ export type Page<T, C> = {
/** Backend-owned JSON continuation state that callers only round-trip to the same scan. */
export type Cursor = Readonly<Record<string, JsonValue>>;
/** Optional filters for an ordered conversation scan. */
export type ConversationQuery = {
readonly ownerConversationId?: ConversationId;
readonly ownerTaskId?: TaskId;
};
/** Inclusive ID bounds for a newest-first scan of one conversation's fork-aware history. */
export type EntryQuery = {
readonly conversationId: Id;
readonly conversationId: ConversationId;
/** Oldest entry ID that may be returned. */
readonly minEntryId?: Id;
readonly minEntryId?: EntryId;
/** Newest entry ID that may be returned. */
readonly maxEntryId?: Id;
readonly maxEntryId?: EntryId;
};
/** Optional filters for an ordered scan of durable task records. */
export type TaskQuery = {
readonly conversationId?: Id;
readonly conversationId?: ConversationId;
readonly kind?: string;
readonly status?: "pending" | "running" | "terminal";
readonly abortRequested?: boolean;
@ -475,6 +496,12 @@ export type DocumentContent =
readonly ops: readonly Op[];
};
/** Exact persisted source selected for a definition-free document copy. */
export type DocumentCopySource = {
readonly id: DocumentId;
readonly at: DocumentPoint;
};
/** Detached materialized value and stored definition version at a selected point. */
export type StoredDocument = {
readonly record: DocumentRecord;
@ -493,22 +520,27 @@ export type StorageWrite =
readonly record: DocumentCreate;
readonly content: Extract<DocumentContent, { readonly kind: "base" }>;
}
| { readonly type: "document.copy"; readonly record: DocumentCreate; readonly source: DocumentCopySource }
| {
readonly type: "document.change";
readonly id: Id;
readonly id: DocumentId;
readonly content: DocumentContent;
}
| { readonly type: "document.retire"; readonly id: Id };
| { readonly type: "document.retire"; readonly id: DocumentId };
/**
* Transaction surface of one Session commit callback.
* Table reads and creation results are trusted immutable values and may be shared with internal commit state.
*/
export interface Tx {
conversation(id: Id): Promise<ConversationRecord | undefined>;
entry(id: Id): Promise<EntryRecord | undefined>;
task(id: Id): Promise<TaskRecord<JsonValue, JsonValue, JsonValue> | undefined>;
scanConversations(limit: number, cursor?: Cursor): Promise<Page<ConversationRecord, Cursor>>;
conversation(id: ConversationId): Promise<ConversationRecord | undefined>;
entry(id: EntryId): Promise<EntryRecord | undefined>;
task(id: TaskId): Promise<TaskRecord<JsonValue, JsonValue, JsonValue> | undefined>;
scanConversations(
query: ConversationQuery,
limit: number,
cursor?: Cursor,
): Promise<Page<ConversationRecord, Cursor>>;
scanEntries(query: EntryQuery, limit: number, cursor?: Cursor): Promise<Page<EntryRecord, Cursor>>;
scanTasks(
query: TaskQuery,
@ -516,21 +548,27 @@ export interface Tx {
cursor?: Cursor,
): Promise<Page<TaskRecord<JsonValue, JsonValue, JsonValue>, Cursor>>;
/** Create a conversation with explicitly selected ownership. */
createConversation(options: { readonly ownership: ConversationOwnership }): Promise<ConversationRecord>;
/** Create a history fork at one concrete visible entry with explicitly selected ownership. */
forkConversation(
parentConversationId: ConversationId,
at: EntryId,
options: { readonly ownership: ConversationOwnership },
): Promise<ConversationRecord>;
/** Returned records are Session-owned immutable values and may be shared with commit listeners. */
createConversation(value: Omit<ConversationRecord, "id">): Promise<ConversationRecord>;
/** Returned records are Session-owned immutable values and may be shared with commit listeners. */
appendEntry(conversationId: Id, value: EntryDraft): Promise<EntryRecord>;
appendEntry(conversationId: ConversationId, value: EntryDraft): Promise<EntryRecord>;
createTask<I, S extends { phase: string }, R, H extends object>(
task: Task<I, S, R, H>,
input: I,
options?: TaskOptions,
): Promise<TaskRef<R>>;
): Promise<TaskId<R>>;
/** Replace one task record completely. */
setTask(value: TaskRecord<JsonValue, JsonValue, JsonValue>): void;
doc<T extends JsonObject>(token: SessionDocToken<T>): Promise<Draft<T>>;
doc<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: Id): Promise<Draft<T>>;
doc<T extends JsonObject>(token: TaskDocToken<T>, taskId: Id): Promise<Draft<T>>;
doc<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: ConversationId): Promise<Draft<T>>;
doc<T extends JsonObject>(token: TaskDocToken<T>, taskId: TaskId): Promise<Draft<T>>;
doc<T extends JsonObject, I extends JsonValue>(
token: SessionDocFamilyToken<T, I>,
key: string,
@ -538,29 +576,29 @@ export interface Tx {
): Promise<Draft<T>>;
doc<T extends JsonObject, I extends JsonValue>(
token: ConversationDocFamilyToken<T, I>,
conversationId: Id,
conversationId: ConversationId,
key: string,
seed: I,
): Promise<Draft<T>>;
doc<T extends JsonObject, I extends JsonValue>(
token: TaskDocFamilyToken<T, I>,
taskId: Id,
taskId: TaskId,
key: string,
seed: I,
): Promise<Draft<T>>;
retireDoc<T extends JsonObject>(token: SessionDocToken<T>): Promise<void>;
retireDoc<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: Id): Promise<void>;
retireDoc<T extends JsonObject>(token: TaskDocToken<T>, taskId: Id): Promise<void>;
retireDoc<T extends JsonObject>(token: ConversationDocToken<T>, conversationId: ConversationId): Promise<void>;
retireDoc<T extends JsonObject>(token: TaskDocToken<T>, taskId: TaskId): Promise<void>;
retireDoc<T extends JsonObject, I extends JsonValue>(token: SessionDocFamilyToken<T, I>, key: string): Promise<void>;
retireDoc<T extends JsonObject, I extends JsonValue>(
token: ConversationDocFamilyToken<T, I>,
conversationId: Id,
conversationId: ConversationId,
key: string,
): Promise<void>;
retireDoc<T extends JsonObject, I extends JsonValue>(
token: TaskDocFamilyToken<T, I>,
taskId: Id,
taskId: TaskId,
key: string,
): Promise<void>;
}
@ -575,12 +613,12 @@ export interface Session {
snapshot<T extends JsonObject>(token: SessionDocToken<T>, context: Context): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject>(
token: ConversationDocToken<T>,
conversationId: Id,
conversationId: ConversationId,
context: Context,
): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject>(
token: TaskDocToken<T>,
taskId: Id,
taskId: TaskId,
context: Context,
): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject, I extends JsonValue>(
@ -590,28 +628,28 @@ export interface Session {
): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject, I extends JsonValue>(
token: ConversationDocFamilyToken<T, I>,
conversationId: Id,
conversationId: ConversationId,
key: string,
context: Context,
): Promise<Readonly<T> | undefined>;
snapshot<T extends JsonObject, I extends JsonValue>(
token: TaskDocFamilyToken<T, I>,
taskId: Id,
taskId: TaskId,
key: string,
context: Context,
): Promise<Readonly<T> | undefined>;
snapshotAsOf<T extends JsonObject>(
token: RewindableConversationDocToken<T>,
conversationId: Id,
at: Id,
conversationId: ConversationId,
at: EntryId,
context: Context,
): Promise<Readonly<T> | undefined>;
snapshotAsOf<T extends JsonObject, I extends JsonValue>(
token: RewindableConversationDocFamilyToken<T, I>,
conversationId: Id,
conversationId: ConversationId,
key: string,
at: Id,
at: EntryId,
context: Context,
): Promise<Readonly<T> | undefined>;
}
@ -630,25 +668,26 @@ export interface Storage {
*/
commit(writes: readonly StorageWrite[], context: Context): Promise<Seq>;
/** Return a fresh candidate from the Session-global record ID namespace. */
mintId(): Promise<Id>;
/** Return a fresh branded candidate from the Session-global numeric ID namespace. */
mintId<I extends Id<string>>(): Promise<I>;
/** Look up one conversation by exact ID. */
conversation(id: Id, context: Context): Promise<ConversationRecord | undefined>;
conversation(id: ConversationId, context: Context): Promise<ConversationRecord | undefined>;
/** Scan conversations in ascending ID order. */
scanConversations(
query: ConversationQuery,
limit: number,
cursor: Cursor | undefined,
context: Context,
): Promise<Page<ConversationRecord, Cursor>>;
/** Look up one global entry and the sequence of the commit that persisted it. */
entry(id: Id, context: Context): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
entry(id: EntryId, context: Context): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
/** Look up one entry only when it is visible through the requested conversation's ancestry. */
entry(
conversationId: Id,
id: Id,
conversationId: ConversationId,
id: EntryId,
context: Context,
): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>;
@ -657,10 +696,10 @@ export interface Storage {
* The returned entry is the marker; its `head` value is the range's actual lower bound.
*/
findLatestHeadMarker(
conversationId: Id,
atOrBeforeEntryId: Id | undefined,
conversationId: ConversationId,
atOrBeforeEntryId: EntryId | undefined,
context: Context,
): Promise<(EntryRecord & { readonly head: Id }) | undefined>;
): Promise<(EntryRecord & { readonly head: EntryId }) | undefined>;
/** Scan the inclusive visible range newest-first, returning at most `limit` entries. */
scanEntries(
@ -671,7 +710,7 @@ export interface Storage {
): Promise<Page<EntryRecord, Cursor>>;
/** Look up the latest complete record for one task. */
task(id: Id, context: Context): Promise<TaskRecord<JsonValue, JsonValue, JsonValue> | undefined>;
task(id: TaskId, context: Context): Promise<TaskRecord<JsonValue, JsonValue, JsonValue> | undefined>;
/** Scan task records matching every supplied filter. */
scanTasks(
@ -682,16 +721,20 @@ export interface Storage {
): Promise<Page<TaskRecord<JsonValue, JsonValue, JsonValue>, Cursor>>;
/** Look up the latest complete record for one admitted submission. */
submission(id: Id, context: Context): Promise<SubmissionRecord | undefined>;
submission(id: SubmissionId, context: Context): Promise<SubmissionRecord | undefined>;
/** Find a submission by its conversation-scoped host deduplication key. */
submissionByRequest(conversationId: Id, requestId: string, context: Context): Promise<SubmissionRecord | undefined>;
submissionByRequest(
conversationId: ConversationId,
requestId: string,
context: Context,
): Promise<SubmissionRecord | undefined>;
/** Resolve the incarnation occupying one exact logical address at the selected point. */
findDocument(address: DocumentAddress, at: DocumentPoint, context: Context): Promise<DocumentRecord | undefined>;
/** Materialize one specific incarnation by ID at the selected point without following a replacement at its address. */
document(id: Id, at: DocumentPoint, context: Context): Promise<StoredDocument | undefined>;
document(id: DocumentId, at: DocumentPoint, context: Context): Promise<StoredDocument | undefined>;
/** Scan incarnations alive in one exact scope at the selected point. */
scanDocuments(

View file

@ -7,9 +7,21 @@ import { registerStorageConformance } from "@earendil-works/pi-durable/testing";
import { afterEach, describe, expect, it } from "vitest";
import { err, FileError, type FileSystem, type Result } from "../src/env/index.ts";
import { NodeExecutionEnv } from "../src/env/node.ts";
import { idFromNumber } from "../src/ids.ts";
import { JsonlStorage } from "../src/storage/jsonl/index.ts";
import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts";
import type { DocumentCreate, Seq, Storage, StorageWrite, TaskRecord } from "../src/types.ts";
import type {
ConversationId,
DocumentCreate,
DocumentId,
EntryId,
Id,
Seq,
Storage,
StorageWrite,
TaskId,
TaskRecord,
} from "../src/types.ts";
import { ROOT_CONVERSATION_ID } from "../src/types.ts";
const context = BACKGROUND_CONTEXT;
@ -64,16 +76,24 @@ class ReopeningStorage implements Storage {
}
}
mintId: Storage["mintId"] = () => this.current.mintId();
mintId<I extends Id<string>>(): Promise<I> {
return this.current.mintId<I>();
}
conversation: Storage["conversation"] = (id, readContext) => this.current.conversation(id, readContext);
scanConversations: Storage["scanConversations"] = (limit, cursor, readContext) =>
this.current.scanConversations(limit, cursor, readContext);
entry(id: number, readContext: Context): ReturnType<Storage["entry"]>;
entry(conversationId: number, id: number, readContext: Context): ReturnType<Storage["entry"]>;
entry(idOrConversationId: number, idOrContext: number | Context, readContext?: Context) {
return readContext === undefined
? this.current.entry(idOrConversationId, idOrContext as Context)
: this.current.entry(idOrConversationId, idOrContext as number, readContext);
scanConversations: Storage["scanConversations"] = (query, limit, cursor, readContext) =>
this.current.scanConversations(query, limit, cursor, readContext);
entry(id: EntryId, readContext: Context): ReturnType<Storage["entry"]>;
entry(conversationId: ConversationId, id: EntryId, readContext: Context): ReturnType<Storage["entry"]>;
entry(idOrConversationId: EntryId | ConversationId, idOrContext: EntryId | Context, readContext?: Context) {
if (readContext === undefined) {
return this.current.entry(idFromNumber<EntryId>(idOrConversationId), idOrContext as Context);
}
if (typeof idOrContext !== "number") throw new TypeError("Storage.entry() requires an entry ID");
return this.current.entry(
idFromNumber<ConversationId>(idOrConversationId),
idFromNumber<EntryId>(idOrContext),
readContext,
);
}
findLatestHeadMarker: Storage["findLatestHeadMarker"] = (conversationId, at, readContext) =>
this.current.findLatestHeadMarker(conversationId, at, readContext);
@ -111,7 +131,7 @@ registerStorageConformance({ describe, expect, it }, "JsonlStorage across reopen
}
});
function pendingTask(id: number, phase = "ready"): StoredTask {
function pendingTask(id: TaskId<JsonValue>, phase = "ready"): StoredTask {
return {
id,
conversationId: ROOT_CONVERSATION_ID,
@ -125,7 +145,7 @@ function pendingTask(id: number, phase = "ready"): StoredTask {
};
}
function terminalTask(id: number): StoredTask {
function terminalTask(id: TaskId<JsonValue>): StoredTask {
return {
id,
conversationId: ROOT_CONVERSATION_ID,
@ -143,7 +163,7 @@ async function createRoot(storage: Storage): Promise<void> {
await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context);
}
function sessionDocument(id: number, kind = "test.document"): DocumentCreate {
function sessionDocument(id: DocumentId, kind = "test.document"): DocumentCreate {
return { id, kind, scope: { kind: "session" } };
}
@ -307,8 +327,8 @@ describe("Pico JsonlStorage publication and recovery", () => {
const directory = await tempDirectory();
const storage = await openStorage(directory);
await createRoot(storage);
const taskId = await storage.mintId();
const documentId = await storage.mintId();
const taskId = await storage.mintId<TaskId<JsonValue>>();
const documentId = await storage.mintId<DocumentId>();
await storage.commit([{ type: "task", value: pendingTask(taskId) }], context);
await storage.commit(
[
@ -353,7 +373,7 @@ describe("Pico JsonlStorage publication and recovery", () => {
const directory = await tempDirectory();
const storage = await openStorage(directory);
await createRoot(storage);
const taskId = await storage.mintId();
const taskId = await storage.mintId<TaskId<JsonValue>>();
await storage.commit(
[
{ type: "task", value: pendingTask(taskId, "first") },
@ -372,7 +392,7 @@ describe("Pico JsonlStorage publication and recovery", () => {
const env = new InstrumentedEnv({ cwd: directory });
const storage = await openStorage(directory, env);
await createRoot(storage);
const taskId = await storage.mintId();
const taskId = await storage.mintId<TaskId<JsonValue>>();
env.fail({ operation: "append", call: 1, mode: "short" });
await expect(
storage.commit(
@ -398,7 +418,7 @@ describe("Pico JsonlStorage publication and recovery", () => {
const storage = await openStorage(directory, env);
await createRoot(storage);
env.clear();
const id = await storage.mintId();
const id = await storage.mintId<EntryId>();
await expect(
storage.commit(
[
@ -440,8 +460,8 @@ describe("Pico JsonlStorage publication and recovery", () => {
const env = new InstrumentedEnv({ cwd: directory });
const storage = await openStorage(directory, env);
await createRoot(storage);
const firstId = await storage.mintId();
const secondId = await storage.mintId();
const firstId = await storage.mintId<DocumentId>();
const secondId = await storage.mintId<DocumentId>();
env.fail(failure);
await expect(
storage.commit(
@ -484,8 +504,8 @@ describe("Pico JsonlStorage publication and recovery", () => {
const env = new InstrumentedEnv({ cwd: directory });
const storage = await openStorage(directory, env, { fsync: true });
await createRoot(storage);
const firstId = await storage.mintId();
const secondId = await storage.mintId();
const firstId = await storage.mintId<DocumentId>();
const secondId = await storage.mintId<DocumentId>();
env.fail(failure);
await expect(
storage.commit(
@ -516,8 +536,8 @@ describe("Pico JsonlStorage publication and recovery", () => {
const env = new InstrumentedEnv({ cwd: directory });
const storage = await openStorage(directory, env, { fsync });
await createRoot(storage);
const firstId = await storage.mintId();
const secondId = await storage.mintId();
const firstId = await storage.mintId<DocumentId>();
const secondId = await storage.mintId<DocumentId>();
env.clear();
await storage.commit(
[
@ -569,7 +589,7 @@ describe("Pico JsonlStorage publication and recovery", () => {
{
type: "entry",
value: {
id: await storage.mintId(),
id: await storage.mintId<EntryId>(),
conversationId: ROOT_CONVERSATION_ID,
kind: "main-only",
},
@ -579,7 +599,7 @@ describe("Pico JsonlStorage publication and recovery", () => {
);
expect(env.operations).toEqual(["append:main.jsonl"]);
const taskId = await storage.mintId();
const taskId = await storage.mintId<TaskId<JsonValue>>();
await storage.commit([{ type: "task", value: pendingTask(taskId) }], context);
env.clear();
await storage.commit([{ type: "task", value: terminalTask(taskId) }], context);
@ -603,7 +623,7 @@ describe("Pico JsonlStorage publication and recovery", () => {
const env = new InstrumentedEnv({ cwd: directory });
const storage = await openStorage(directory, env);
await createRoot(storage);
const id = await storage.mintId();
const id = await storage.mintId<DocumentId>();
await storage.commit(
[
{
@ -654,8 +674,8 @@ describe("Pico JsonlStorage publication and recovery", () => {
const env = new InstrumentedEnv({ cwd: directory });
const storage = await openStorage(directory, env);
await createRoot(storage);
const taskId = await storage.mintId();
const id = await storage.mintId();
const taskId = await storage.mintId<TaskId<JsonValue>>();
const id = await storage.mintId<DocumentId>();
await storage.commit(
[
{ type: "task", value: pendingTask(taskId) },
@ -686,7 +706,7 @@ describe("Pico JsonlStorage publication and recovery", () => {
const env = new InstrumentedEnv({ cwd: directory });
const storage = await openStorage(directory, env, { fsync: true });
await createRoot(storage);
const id = await storage.mintId();
const id = await storage.mintId<DocumentId>();
await storage.commit(
[
{
@ -740,7 +760,7 @@ describe("Pico JsonlStorage publication and recovery", () => {
const env = new InstrumentedEnv({ cwd: directory });
const storage = await openStorage(directory, env, { fsync: true });
await createRoot(storage);
const id = await storage.mintId();
const id = await storage.mintId<DocumentId>();
await storage.commit(
[
{
@ -789,7 +809,7 @@ describe("Pico JsonlStorage publication and recovery", () => {
const env = new InstrumentedEnv({ cwd: directory });
const storage = await openStorage(directory, env);
await createRoot(storage);
const id = await storage.mintId();
const id = await storage.mintId<TaskId<JsonValue>>();
await storage.commit([{ type: "task", value: pendingTask(id) }], context);
env.fail({ operation: "remove", call: 1, mode });
await expect(storage.commit([{ type: "task", value: terminalTask(id) }], context)).resolves.toBe(3);
@ -807,7 +827,7 @@ describe("Pico JsonlStorage publication and recovery", () => {
const directory = await tempDirectory();
const storage = await openStorage(directory);
await createRoot(storage);
const id = await storage.mintId();
const id = await storage.mintId<DocumentId>();
await storage.commit(
[
{
@ -847,7 +867,7 @@ describe("Pico JsonlStorage publication and recovery", () => {
const directory = await tempDirectory();
const storage = await openStorage(directory);
await createRoot(storage);
const id = await storage.mintId();
const id = await storage.mintId<DocumentId>();
const record = {
id,
kind: "rewindable",
@ -893,10 +913,10 @@ describe("Pico JsonlStorage publication and recovery", () => {
const directory = await tempDirectory();
const storage = await openStorage(directory);
await createRoot(storage);
const taskId = await storage.mintId();
const sessionId = await storage.mintId();
const latestId = await storage.mintId();
const taskDocumentId = await storage.mintId();
const taskId = await storage.mintId<TaskId<JsonValue>>();
const sessionId = await storage.mintId<DocumentId>();
const latestId = await storage.mintId<DocumentId>();
const taskDocumentId = await storage.mintId<DocumentId>();
const createdAt = await storage.commit(
[
{ type: "task", value: pendingTask(taskId) },
@ -959,7 +979,7 @@ describe("Pico JsonlStorage publication and recovery", () => {
const directory = await tempDirectory();
const storage = await openStorage(directory);
await createRoot(storage);
const documentId = await storage.mintId();
const documentId = await storage.mintId<DocumentId>();
await storage.commit(
[
{
@ -993,7 +1013,7 @@ describe("Pico JsonlStorage publication and recovery", () => {
const directory = await tempDirectory();
const storage = await openStorage(directory);
await createRoot(storage);
const taskId = await storage.mintId();
const taskId = await storage.mintId<TaskId<JsonValue>>();
await storage.commit([{ type: "task", value: pendingTask(taskId) }], context);
await storage.commit([{ type: "task", value: terminalTask(taskId) }], context);
const sidecarPath = join(directory, `task-${taskId}.jsonl`);
@ -1020,7 +1040,7 @@ describe("Pico JsonlStorage publication and recovery", () => {
const directory = await tempDirectory();
const storage = await openStorage(directory);
await createRoot(storage);
const documentId = await storage.mintId();
const documentId = await storage.mintId<DocumentId>();
await storage.commit(
[
{
@ -1041,7 +1061,7 @@ describe("Pico JsonlStorage publication and recovery", () => {
const directory = await tempDirectory();
const storage = await openStorage(directory);
await createRoot(storage);
const documentId = await storage.mintId();
const documentId = await storage.mintId<DocumentId>();
await storage.commit(
[
{
@ -1093,7 +1113,7 @@ describe("Pico JsonlStorage publication and recovery", () => {
const directory = await tempDirectory();
const storage = await openStorage(directory);
await createRoot(storage);
const documentId = await storage.mintId();
const documentId = await storage.mintId<DocumentId>();
await storage.commit(
[
{

View file

@ -1,19 +1,21 @@
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
import { registerStorageConformance } from "@earendil-works/pi-durable/testing";
import { describe, expect, it } from "vitest";
import { idFromNumber, seqFromNumber } from "../src/ids.ts";
import { MemoryStorage } from "../src/storage/memory.ts";
import { ROOT_CONVERSATION_ID } from "../src/types.ts";
import { type EntryId, ROOT_CONVERSATION_ID } from "../src/types.ts";
registerStorageConformance({ describe, expect, it }, "MemoryStorage", (use) => use(new MemoryStorage()));
it("does not expose retained state through a prepared commit", async () => {
const storage = new MemoryStorage();
await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT);
const entryId = idFromNumber<EntryId>(2);
const prepared = storage.prepareCommit([
{
type: "entry",
value: {
id: 2,
id: entryId,
conversationId: ROOT_CONVERSATION_ID,
kind: "test",
data: { nested: [1] },
@ -24,7 +26,7 @@ it("does not expose retained state through a prepared commit", async () => {
if (exposed.type !== "entry") throw new Error("Expected an entry write");
expect(() => (exposed.value.data as { nested: number[] }).nested.push(2)).toThrow();
expect(prepared.apply()).toBe(2);
expect(prepared.apply()).toBe(2);
expect((await storage.entry(2, BACKGROUND_CONTEXT))?.entry.data).toEqual({ nested: [1] });
expect(prepared.apply()).toBe(seqFromNumber(2));
expect(prepared.apply()).toBe(seqFromNumber(2));
expect((await storage.entry(entryId, BACKGROUND_CONTEXT))?.entry.data).toEqual({ nested: [1] });
});

View file

@ -0,0 +1,140 @@
// A tour of the durable Session API in four small examples.
// Run from packages/durable:
// node --conditions=source --experimental-strip-types test/scratch.ts
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
import { type ConversationId, createSession, defineDoc, MemoryStorage, type Task } from "../src/index.ts";
// A Session stores conversations, transcript entries, tasks, and documents.
// MemoryStorage keeps everything in memory; other storage backends keep it on disk.
const session = createSession(new MemoryStorage());
// Every Session call takes a context, which is used for cancellation.
// BACKGROUND_CONTEXT means "never cancel".
const context = BACKGROUND_CONTEXT;
// ─── 1. Create a standalone conversation ────────────────────────────────────
// All writes happen inside session.commit(). The callback receives a
// transaction `tx`; everything it writes is saved together when the callback
// returns, or discarded if it throws.
// "ownerless" means no task created this conversation.
const standalone = await session.commit((tx) => tx.createConversation({ ownership: { kind: "ownerless" } }), context);
console.log("1. standalone conversation:", standalone);
// ─── 2. Store document state next to transcript entries ─────────────────────
// A document is a JSON object attached to something; here, one per conversation.
// "rewindable" keeps old values readable, so you can ask what the document
// looked like when a particular entry was written.
// `fork` says what a forked copy of the conversation starts with (see example 3).
const Notes = defineDoc<{ text: string }>({
kind: "example.notes",
version: 1,
scope: "conversation",
history: "rewindable",
fork: "asOf", // a fork starts with the value these notes had at the fork entry
initial: () => ({ text: "" }),
});
const chat = await session.commit((tx) => tx.createConversation({ ownership: { kind: "ownerless" } }), context);
// tx.doc() returns an editable copy of the document (created on first use).
// Plain assignments to it are saved when the commit finishes.
const firstEntry = await session.commit(async (tx) => {
const entry = await tx.appendEntry(chat.id, { kind: "note", data: "hello" });
(await tx.doc(Notes, chat.id)).text = "after hello";
return entry;
}, context);
const secondEntry = await session.commit(async (tx) => {
const entry = await tx.appendEntry(chat.id, { kind: "note", data: "goodbye" });
(await tx.doc(Notes, chat.id)).text = "after goodbye";
return entry;
}, context);
// snapshot() reads the latest value. snapshotAsOf() reads the value that was
// saved in the same commit as the given entry.
console.log("2. latest notes:", await session.snapshot(Notes, chat.id, context));
console.log("2. notes at first entry:", await session.snapshotAsOf(Notes, chat.id, firstEntry.id, context));
console.log("2. notes at second entry:", await session.snapshotAsOf(Notes, chat.id, secondEntry.id, context));
// ─── 3. Fork a conversation ─────────────────────────────────────────────────
// A fork is a new conversation that continues from one entry of another. It
// sees the parent's transcript up to that entry, and each document follows its
// own `fork` setting. Notes uses "asOf", so the fork starts with the notes
// value from the fork entry.
const branch = await session.commit(
(tx) => tx.forkConversation(chat.id, firstEntry.id, { ownership: { kind: "ownerless" } }),
context,
);
// scanEntries() pages through visible entries, newest first. The fork sees
// "hello" (inherited from the parent) but not "goodbye", which came later.
const branchEntries = await session.commit((tx) => tx.scanEntries({ conversationId: branch.id }, 10), context);
console.log(
"3. fork transcript:",
branchEntries.items.map((entry) => entry.data),
);
console.log("3. fork notes:", await session.snapshot(Notes, branch.id, context));
// The fork's copy is independent: editing it leaves the parent unchanged.
await session.commit(async (tx) => {
(await tx.doc(Notes, branch.id)).text = "changed only in the fork";
}, context);
console.log("3. fork notes after edit:", await session.snapshot(Notes, branch.id, context));
console.log("3. parent notes after edit:", await session.snapshot(Notes, chat.id, context));
// ─── 4. Background task that owns a child conversation ──────────────────────
// A typical agent setup: a background task supervises a helper conversation,
// and the main conversation keeps a registry that maps agent names to their
// conversations. All three are created in one commit, so after a crash either
// all of them exist or none do.
// A task definition needs a name, a version, and the task's starting state.
// This example only creates the task record; nothing runs it yet.
const Supervisor: Task<null, { phase: "ready" }, null, object> = {
definition: {
name: "example.supervisor",
version: 1,
initial: () => ({ phase: "ready" }),
},
};
// "latest" keeps only the current value. "initial" means forks of this
// conversation start without a registry, so a child doesn't inherit its
// parent's list of agents.
const AgentRegistry = defineDoc<{
agents: Record<string, { conversationId: ConversationId; requestId: string }>;
}>({
kind: "example.agent-registry",
version: 1,
scope: "conversation",
history: "latest",
fork: "initial",
initial: () => ({ agents: {} }),
});
const main = await session.commit((tx) => tx.createConversation({ ownership: { kind: "ownerless" } }), context);
const setup = await session.commit(async (tx) => {
// `background: true` means the task is side work: waiting for the main
// conversation to finish does not wait for it.
const supervisorId = await tx.createTask(Supervisor, null, { conversationId: main.id, background: true });
// The child records that it belongs to the supervisor task. The task was
// created a few lines above in this same commit, which is allowed.
const child = await tx.createConversation({ ownership: { kind: "task", taskId: supervisorId } });
// requestId is a fixed name for the child's first message. Later code sends
// that message using this requestId, so a retry after a crash cannot
// deliver it twice.
(await tx.doc(AgentRegistry, main.id)).agents.researcher = {
conversationId: child.id,
requestId: `researcher:first-message:${supervisorId}`,
};
return { supervisorId, child };
}, context);
console.log("4. supervisor task:", setup.supervisorId);
console.log("4. child conversation:", setup.child);
console.log("4. registry:", await session.snapshot(AgentRegistry, main.id, context));
await session.close(context);

View file

@ -1,5 +1,11 @@
import type { Op } from "@earendil-works/chord/delta";
import { defineDoc, defineDocFamily, type Id, type JsonObject, type StorageWrite } from "@earendil-works/pi-durable";
import {
defineDoc,
defineDocFamily,
type EntryId,
type JsonObject,
type StorageWrite,
} from "@earendil-works/pi-durable";
import { describe, expect, it } from "vitest";
import { context, createConversation, documentChanges, flush, openTestSession } from "./session-support.ts";
@ -585,9 +591,9 @@ describe("Session historical document snapshots", () => {
});
const { session, storage } = openTestSession();
const conversationId = await createConversation(session);
let firstEntry!: Id;
let secondEntry!: Id;
let thirdEntry!: Id;
let firstEntry!: EntryId;
let secondEntry!: EntryId;
let thirdEntry!: EntryId;
await session.commit(async (tx) => {
firstEntry = (await tx.appendEntry(conversationId, { kind: "first" })).id;
(await tx.doc(V1, conversationId)).count = 1;
@ -625,11 +631,7 @@ describe("Session historical document snapshots", () => {
const childId = await session.commit(
async (tx) =>
(
await tx.createConversation({
parent: { conversationId, at: secondEntry },
})
).id,
(await tx.forkConversation(conversationId, secondEntry, { ownership: { kind: "ownerless" } })).id,
context,
);
expect(await session.snapshotAsOf(V3, childId, firstEntry, context)).toEqual({ count: 1, version: 3 });
@ -658,9 +660,9 @@ describe("Session historical document snapshots", () => {
async (tx) => (await tx.appendEntry(conversationId, { kind: "before" })).id,
context,
);
let createdAt!: Id;
let retiredAt!: Id;
let recreatedAt!: Id;
let createdAt!: EntryId;
let retiredAt!: EntryId;
let recreatedAt!: EntryId;
await session.commit(async (tx) => {
createdAt = (await tx.appendEntry(conversationId, { kind: "create" })).id;
(await tx.doc(Doc, conversationId)).value = "old";

View file

@ -1,14 +1,17 @@
import type { Draft } from "@earendil-works/chord";
import {
type ConversationId,
createSession,
defineDoc,
defineDocFamily,
type Id,
MemoryStorage,
ROOT_CONVERSATION_ID,
type Session,
type TaskId,
type Tx,
} from "@earendil-works/pi-durable";
import { describe, expect, expectTypeOf, it } from "vitest";
import { idFromNumber } from "../src/ids.ts";
import { context } from "./session-support.ts";
type State = { value: number };
@ -68,7 +71,7 @@ describe("document definitions", () => {
it("types every owner, key, and seed overload", async () => {
const session: Session = createSession(new MemoryStorage());
const check = async (tx: Tx, conversationId: Id, taskId: Id): Promise<void> => {
const check = async (tx: Tx, conversationId: ConversationId, taskId: TaskId): Promise<void> => {
expectTypeOf(await tx.doc(SessionDoc)).toEqualTypeOf<Draft<State>>();
expectTypeOf(await tx.doc(LatestDoc, conversationId)).toEqualTypeOf<Draft<State>>();
expectTypeOf(await tx.doc(RewindableDoc, conversationId)).toEqualTypeOf<Draft<State>>();
@ -97,13 +100,18 @@ describe("document definitions", () => {
expect(check).toBeTypeOf("function");
expectTypeOf(await session.snapshot(SessionDoc, context)).toEqualTypeOf<Readonly<State> | undefined>();
expectTypeOf(await session.snapshot(LatestDoc, 1, context)).toEqualTypeOf<Readonly<State> | undefined>();
expectTypeOf(await session.snapshot(TaskDoc, 1, context)).toEqualTypeOf<Readonly<State> | undefined>();
expectTypeOf(await session.snapshot(SessionFamily, "k", context)).toEqualTypeOf<Readonly<State> | undefined>();
expectTypeOf(await session.snapshot(ConversationFamily, 1, "k", context)).toEqualTypeOf<
const taskId = idFromNumber<TaskId>(1);
expectTypeOf(await session.snapshot(LatestDoc, ROOT_CONVERSATION_ID, context)).toEqualTypeOf<
Readonly<State> | undefined
>();
expectTypeOf(await session.snapshot(TaskDoc, taskId, context)).toEqualTypeOf<Readonly<State> | undefined>();
expectTypeOf(await session.snapshot(SessionFamily, "k", context)).toEqualTypeOf<Readonly<State> | undefined>();
expectTypeOf(await session.snapshot(ConversationFamily, ROOT_CONVERSATION_ID, "k", context)).toEqualTypeOf<
Readonly<State> | undefined
>();
expectTypeOf(await session.snapshot(TaskFamily, taskId, "k", context)).toEqualTypeOf<
Readonly<State> | undefined
>();
expectTypeOf(await session.snapshot(TaskFamily, 1, "k", context)).toEqualTypeOf<Readonly<State> | undefined>();
// @ts-expect-error snapshots never take a creation seed
await session.snapshot(SessionFamily, "k", 1, context).catch(() => undefined);
await session.close(context);

View file

@ -1,6 +1,13 @@
import type { Draft } from "@earendil-works/chord";
import { defineDoc, defineDocFamily, type Id, type JsonObject } from "@earendil-works/pi-durable";
import type { Draft, JsonValue } from "@earendil-works/chord";
import {
type ConversationId,
defineDoc,
defineDocFamily,
type JsonObject,
type TaskId,
} from "@earendil-works/pi-durable";
import { describe, expect, it } from "vitest";
import { idFromNumber } from "../src/ids.ts";
import { context, createConversation, documentChanges, flush, openTestSession } from "./session-support.ts";
type Live = { message?: string; items: string[]; nested: { count: number }; other: { label: string } };
@ -57,7 +64,7 @@ const MemberDoc = defineDocFamily<Member, string>({
},
});
async function setupLive(): Promise<ReturnType<typeof openTestSession> & { readonly conversationId: Id }> {
async function setupLive(): Promise<ReturnType<typeof openTestSession> & { readonly conversationId: ConversationId }> {
const harness = openTestSession();
const conversationId = await createConversation(harness.session);
await harness.session.commit(async (tx) => {
@ -409,7 +416,7 @@ describe("Session document transactions", () => {
(await tx.doc(CounterDoc)).count = 2;
// Replacing a missing task fails during assembly, after every change was prepared.
tx.setTask({
id: 999,
id: idFromNumber<TaskId<JsonValue>>(999),
conversationId,
kind: "missing",
version: 1,

View file

@ -0,0 +1,694 @@
import {
type ConversationId,
defineDoc,
defineDocFamily,
type EntryId,
StorageRejected,
type StorageWrite,
type TaskId,
} from "@earendil-works/pi-durable";
import { describe, expect, it } from "vitest";
import {
context,
createConversation,
documentChanges,
documentCopyChanges,
flush,
openTestSession,
} from "./session-support.ts";
function documentCreates(writes: readonly StorageWrite[]) {
return writes.filter((write) => write.type === "document.create");
}
function documentCopies(writes: readonly StorageWrite[]) {
return writes.filter((write) => write.type === "document.copy");
}
describe("Session conversation document forks", () => {
it("copies as-of and current singleton and family bases while leaving initial documents absent", async () => {
const AsOf = defineDoc<{ value: string }>({
kind: "fork.policies.as-of",
version: 1,
scope: "conversation",
history: "rewindable",
fork: "asOf",
initial: () => ({ value: "as-initial" }),
});
const Current = defineDoc<{ value: string }>({
kind: "fork.policies.current",
version: 1,
scope: "conversation",
history: "latest",
fork: "current",
initial: () => ({ value: "current-initial" }),
});
const Initial = defineDoc<{ value: string }>({
kind: "fork.policies.initial",
version: 1,
scope: "conversation",
history: "latest",
fork: "initial",
initial: () => ({ value: "fresh" }),
});
const AsOfFamily = defineDocFamily<{ value: string }, string>({
kind: "fork.policies.as-of-family",
version: 1,
family: true,
scope: "conversation",
history: "rewindable",
fork: "asOf",
initial: (seed) => ({ value: seed }),
});
const CurrentFamily = defineDocFamily<{ value: string }, string>({
kind: "fork.policies.current-family",
version: 1,
family: true,
scope: "conversation",
history: "latest",
fork: "current",
initial: (seed) => ({ value: seed }),
});
const { session, storage, publications } = openTestSession();
const parentId = await createConversation(session);
let forkAt!: EntryId;
await session.commit(async (tx) => {
forkAt = (await tx.appendEntry(parentId, { kind: "fork-point" })).id;
(await tx.doc(AsOf, parentId)).value = "as-at-fork";
(await tx.doc(Current, parentId)).value = "current-at-fork";
(await tx.doc(Initial, parentId)).value = "parent-only";
(await tx.doc(AsOfFamily, parentId, "a", "unused")).value = "family-as-a";
(await tx.doc(AsOfFamily, parentId, "b", "unused")).value = "family-as-b";
(await tx.doc(CurrentFamily, parentId, "a", "unused")).value = "family-current-a";
}, context);
await session.commit(async (tx) => {
(await tx.doc(AsOf, parentId)).value = "as-after-fork";
(await tx.doc(Current, parentId)).value = "current-when-copied";
(await tx.doc(AsOfFamily, parentId, "a", "unused")).value = "family-as-after";
(await tx.doc(CurrentFamily, parentId, "a", "unused")).value = "family-current-when-copied";
}, context);
await flush();
const documentReads = storage.documentReadCount;
const child = await session.commit(
(tx) => tx.forkConversation(parentId, forkAt, { ownership: { kind: "ownerless" } }),
context,
);
await flush();
expect(storage.documentReadCount).toBe(documentReads);
expect(await session.snapshot(AsOf, child.id, context)).toEqual({ value: "as-at-fork" });
expect(await session.snapshot(Current, child.id, context)).toEqual({ value: "current-when-copied" });
expect(await session.snapshot(Initial, child.id, context)).toBeUndefined();
expect(await session.snapshot(AsOfFamily, child.id, "a", context)).toEqual({ value: "family-as-a" });
expect(await session.snapshot(AsOfFamily, child.id, "b", context)).toEqual({ value: "family-as-b" });
expect(await session.snapshot(CurrentFamily, child.id, "a", context)).toEqual({
value: "family-current-when-copied",
});
const admitted = storage.admittedCommits.at(-1)!;
const copies = documentCopies(admitted);
expect(copies).toHaveLength(5);
expect(copies.every((write) => write.record.scope.kind === "conversation")).toBe(true);
expect(
copies.every(
(write) => write.record.scope.kind === "conversation" && write.record.scope.conversationId === child.id,
),
).toBe(true);
const publication = publications.at(-1)!;
const copied = documentCopyChanges(publication);
expect(copied).toHaveLength(5);
for (const change of copied) {
expect(change.record.createdAt).toBe(publication.seq);
expect(change.conversationId).toBe(child.id);
const write = copies.find((candidate) => candidate.record.id === change.record.id)!;
expect(change.source).toEqual(write.source);
}
const parentRecord = await storage.findDocument(
{ kind: AsOf.definition.kind, scope: { kind: "conversation", conversationId: parentId } },
"current",
context,
);
const childRecord = await storage.findDocument(
{ kind: AsOf.definition.kind, scope: { kind: "conversation", conversationId: child.id } },
"current",
context,
);
expect(childRecord!.id).not.toBe(parentRecord!.id);
await session.commit(async (tx) => {
(await tx.doc(AsOf, child.id)).value = "child-independent";
}, context);
expect(await session.snapshot(AsOf, parentId, context)).toEqual({ value: "as-after-fork" });
expect(await session.snapshot(AsOf, child.id, context)).toEqual({ value: "child-independent" });
await session.commit(async (tx) => {
(await tx.doc(Initial, child.id)).value = "child-created";
}, context);
expect(await session.snapshot(Initial, child.id, context)).toEqual({ value: "child-created" });
});
it("uses final document state from the fork entry commit while excluding later same-commit entries", async () => {
const Doc = defineDoc<{ value: string }>({
kind: "fork.same-commit",
version: 1,
scope: "conversation",
history: "rewindable",
fork: "asOf",
initial: () => ({ value: "initial" }),
});
const { session } = openTestSession();
const parentId = await createConversation(session);
let forkAt!: EntryId;
let excluded!: EntryId;
await session.commit(async (tx) => {
forkAt = (await tx.appendEntry(parentId, { kind: "included" })).id;
(await tx.doc(Doc, parentId)).value = "final-state-of-commit";
excluded = (await tx.appendEntry(parentId, { kind: "excluded" })).id;
}, context);
const child = await session.commit(
(tx) => tx.forkConversation(parentId, forkAt, { ownership: { kind: "ownerless" } }),
context,
);
expect(await session.snapshot(Doc, child.id, context)).toEqual({ value: "final-state-of-commit" });
const visible = await session.commit((tx) => tx.scanEntries({ conversationId: child.id }, 10), context);
expect(visible.items.map(({ id }) => id)).toContain(forkAt);
expect(visible.items.map(({ id }) => id)).not.toContain(excluded);
});
it("selects the entry-owning ancestor for as-of copies and the immediate parent for current copies", async () => {
const AsOf = defineDoc<{ value: string }>({
kind: "fork.ancestry.as-of",
version: 1,
scope: "conversation",
history: "rewindable",
fork: "asOf",
initial: () => ({ value: "initial" }),
});
const Current = defineDoc<{ value: string }>({
kind: "fork.ancestry.current",
version: 1,
scope: "conversation",
history: "latest",
fork: "current",
initial: () => ({ value: "initial" }),
});
const { session } = openTestSession();
const rootId = await createConversation(session);
let inherited!: EntryId;
await session.commit(async (tx) => {
inherited = (await tx.appendEntry(rootId, { kind: "root" })).id;
(await tx.doc(AsOf, rootId)).value = "root-at-entry";
(await tx.doc(Current, rootId)).value = "root-current";
}, context);
const parent = await session.commit(
(tx) => tx.forkConversation(rootId, inherited, { ownership: { kind: "ownerless" } }),
context,
);
let parentEntry!: EntryId;
await session.commit(async (tx) => {
parentEntry = (await tx.appendEntry(parent.id, { kind: "parent" })).id;
(await tx.doc(AsOf, parent.id)).value = "parent-at-own-entry";
(await tx.doc(Current, parent.id)).value = "parent-current";
}, context);
const inheritedFork = await session.commit(
(tx) => tx.forkConversation(parent.id, inherited, { ownership: { kind: "ownerless" } }),
context,
);
expect(await session.snapshot(AsOf, inheritedFork.id, context)).toEqual({ value: "root-at-entry" });
expect(await session.snapshot(Current, inheritedFork.id, context)).toEqual({ value: "parent-current" });
const ownEntryFork = await session.commit(
(tx) => tx.forkConversation(parent.id, parentEntry, { ownership: { kind: "ownerless" } }),
context,
);
expect(await session.snapshot(AsOf, ownEntryFork.id, context)).toEqual({ value: "parent-at-own-entry" });
expect(await session.snapshot(Current, ownEntryFork.id, context)).toEqual({ value: "parent-current" });
});
it("copies the stored value and version without consulting migration definitions or migrated caches", async () => {
const V1 = defineDoc<{ count: number }>({
kind: "fork.stored-version",
version: 1,
scope: "conversation",
history: "rewindable",
fork: "asOf",
initial: () => ({ count: 1 }),
});
let migrations = 0;
const V3 = defineDoc<{ count: number; migrated: boolean }>({
kind: "fork.stored-version",
version: 3,
scope: "conversation",
history: "rewindable",
fork: "asOf",
initial: () => ({ count: 0, migrated: false }),
migrate: (value, fromVersion) => {
expect(fromVersion).toBe(1);
migrations++;
return { count: value.count as number, migrated: true };
},
});
const { session, storage } = openTestSession();
const parentId = await createConversation(session);
let forkAt!: EntryId;
await session.commit(async (tx) => {
forkAt = (await tx.appendEntry(parentId, { kind: "point" })).id;
await tx.doc(V1, parentId);
}, context);
await session.unloadDocuments();
expect(await session.snapshot(V3, parentId, context)).toEqual({ count: 1, migrated: true });
expect(migrations).toBe(1);
const child = await session.commit(
(tx) => tx.forkConversation(parentId, forkAt, { ownership: { kind: "ownerless" } }),
context,
);
expect(migrations).toBe(1);
const childRecord = await storage.findDocument(
{ kind: V1.definition.kind, scope: { kind: "conversation", conversationId: child.id } },
"current",
context,
);
const childStored = await storage.document(childRecord!.id, "current", context);
expect(childStored).toMatchObject({ version: 1, value: { count: 1 } });
expect(await session.snapshot(V3, child.id, context)).toEqual({ count: 1, migrated: true });
expect(migrations).toBe(2);
});
it("coalesces a typed migration and override into the copied creation base", async () => {
const V1 = defineDoc<{ count: number }>({
kind: "fork.override",
version: 1,
scope: "conversation",
history: "rewindable",
fork: "asOf",
initial: () => ({ count: 2 }),
});
let checkpoints = 0;
const V2 = defineDoc<{ count: number; migrated: boolean }>({
kind: "fork.override",
version: 2,
scope: "conversation",
history: "rewindable",
fork: "asOf",
initial: () => ({ count: 0, migrated: false }),
migrate: (value) => ({ count: value.count as number, migrated: true }),
checkpointWhen: () => {
checkpoints++;
return false;
},
});
const { session, storage, publications } = openTestSession();
const parentId = await createConversation(session);
let forkAt!: EntryId;
await session.commit(async (tx) => {
forkAt = (await tx.appendEntry(parentId, { kind: "point" })).id;
await tx.doc(V1, parentId);
}, context);
const documentReads = storage.documentReadCount;
const child = await session.commit(async (tx) => {
const created = await tx.forkConversation(parentId, forkAt, { ownership: { kind: "ownerless" } });
const value = await tx.doc(V2, created.id);
value.count = 9;
return created;
}, context);
await flush();
expect(storage.documentReadCount).toBe(documentReads + 1);
const creates = documentCreates(storage.admittedCommits.at(-1)!);
expect(creates).toHaveLength(1);
expect(creates[0]!.content).toMatchObject({
kind: "base",
version: 2,
value: { count: 9, migrated: true },
});
expect(checkpoints).toBe(0);
expect(documentChanges(publications.at(-1)!)).toMatchObject([{ version: 2 }]);
expect(await session.snapshot(V2, child.id, context)).toEqual({ count: 9, migrated: true });
const parentRecord = await storage.findDocument(
{ kind: V1.definition.kind, scope: { kind: "conversation", conversationId: parentId } },
"current",
context,
);
expect((await storage.document(parentRecord!.id, "current", context))!.version).toBe(1);
});
it("copies the incarnation alive at the fork point across retirement and recreation", async () => {
const Doc = defineDoc<{ value: string }>({
kind: "fork.incarnations",
version: 1,
scope: "conversation",
history: "rewindable",
fork: "asOf",
initial: () => ({ value: "initial" }),
});
const { session } = openTestSession();
const parentId = await createConversation(session);
let oldAt!: EntryId;
let retiredAt!: EntryId;
let newAt!: EntryId;
await session.commit(async (tx) => {
oldAt = (await tx.appendEntry(parentId, { kind: "old" })).id;
(await tx.doc(Doc, parentId)).value = "old";
}, context);
await session.commit(async (tx) => {
retiredAt = (await tx.appendEntry(parentId, { kind: "retired" })).id;
await tx.retireDoc(Doc, parentId);
}, context);
await session.commit(async (tx) => {
newAt = (await tx.appendEntry(parentId, { kind: "new" })).id;
(await tx.doc(Doc, parentId)).value = "new";
}, context);
const oldChild = await session.commit(
(tx) => tx.forkConversation(parentId, oldAt, { ownership: { kind: "ownerless" } }),
context,
);
const emptyChild = await session.commit(
(tx) => tx.forkConversation(parentId, retiredAt, { ownership: { kind: "ownerless" } }),
context,
);
const newChild = await session.commit(
(tx) => tx.forkConversation(parentId, newAt, { ownership: { kind: "ownerless" } }),
context,
);
expect(await session.snapshot(Doc, oldChild.id, context)).toEqual({ value: "old" });
expect(await session.snapshot(Doc, emptyChild.id, context)).toBeUndefined();
expect(await session.snapshot(Doc, newChild.id, context)).toEqual({ value: "new" });
});
it("rejects invisible fork points before admission and remains usable", async () => {
const { session, storage, publications } = openTestSession();
const rootId = await createConversation(session);
let visible!: EntryId;
let hidden!: EntryId;
await session.commit(async (tx) => {
visible = (await tx.appendEntry(rootId, { kind: "visible" })).id;
hidden = (await tx.appendEntry(rootId, { kind: "hidden" })).id;
}, context);
const parent = await session.commit(
(tx) => tx.forkConversation(rootId, visible, { ownership: { kind: "ownerless" } }),
context,
);
await flush();
const commits = storage.commits.length;
const published = publications.length;
await expect(
session.commit((tx) => tx.forkConversation(parent.id, hidden, { ownership: { kind: "ownerless" } }), context),
).rejects.toThrow(`Entry ${hidden} is not visible`);
await flush();
expect(storage.commits).toHaveLength(commits);
expect(publications).toHaveLength(published);
const independent = await session.commit(
(tx) => tx.createConversation({ ownership: { kind: "ownerless" } }),
context,
);
expect(await storage.conversation(independent.id, context)).toEqual(independent);
});
it("rejects duplicate as-of and current selections for one child address before admission", async () => {
const AsOf = defineDoc<{ value: string }>({
kind: "fork.duplicate-policy",
version: 1,
scope: "conversation",
history: "rewindable",
fork: "asOf",
initial: () => ({ value: "as-of" }),
});
const Current = defineDoc<{ value: string }>({
kind: "fork.duplicate-policy",
version: 1,
scope: "conversation",
history: "latest",
fork: "current",
initial: () => ({ value: "current" }),
});
const { session, storage } = openTestSession();
const parentId = await createConversation(session);
let oldAt!: EntryId;
await session.commit(async (tx) => {
oldAt = (await tx.appendEntry(parentId, { kind: "old" })).id;
await tx.doc(AsOf, parentId);
}, context);
await session.commit((tx) => tx.retireDoc(AsOf, parentId), context);
await session.commit((tx) => tx.doc(Current, parentId).then(() => undefined), context);
const commits = storage.commits.length;
await expect(
session.commit((tx) => tx.forkConversation(parentId, oldAt, { ownership: { kind: "ownerless" } }), context),
).rejects.toThrow("Fork selects multiple source documents");
expect(storage.commits).toHaveLength(commits);
const independent = await session.commit(
(tx) => tx.createConversation({ ownership: { kind: "ownerless" } }),
context,
);
expect(await storage.conversation(independent.id, context)).toEqual(independent);
});
it("rejects current and as-of source writes in the fork transaction", async () => {
const Current = defineDoc<{ value: string }>({
kind: "fork.same-transaction-current",
version: 1,
scope: "conversation",
history: "latest",
fork: "current",
initial: () => ({ value: "committed" }),
});
const AsOf = defineDoc<{ value: string }>({
kind: "fork.same-transaction-as-of",
version: 1,
scope: "conversation",
history: "rewindable",
fork: "asOf",
initial: () => ({ value: "committed" }),
});
const { session, storage } = openTestSession();
const parentId = await createConversation(session);
let forkAt!: EntryId;
await session.commit(async (tx) => {
forkAt = (await tx.appendEntry(parentId, { kind: "point" })).id;
await tx.doc(Current, parentId);
await tx.doc(AsOf, parentId);
}, context);
const commits = storage.commits.length;
await expect(
session.commit(async (tx) => {
(await tx.doc(Current, parentId)).value = "before-fork";
await tx.forkConversation(parentId, forkAt, { ownership: { kind: "ownerless" } });
}, context),
).rejects.toThrow("Cannot change fork source document");
await expect(
session.commit(async (tx) => {
await tx.forkConversation(parentId, forkAt, { ownership: { kind: "ownerless" } });
(await tx.doc(Current, parentId)).value = "after-fork";
}, context),
).rejects.toThrow("Cannot change fork source document");
await expect(
session.commit(async (tx) => {
(await tx.doc(AsOf, parentId)).value = "as-of-write";
await tx.forkConversation(parentId, forkAt, { ownership: { kind: "ownerless" } });
}, context),
).rejects.toThrow("Cannot change fork source document");
expect(storage.commits).toHaveLength(commits);
expect(await session.snapshot(Current, parentId, context)).toEqual({ value: "committed" });
const child = await session.commit(
(tx) => tx.forkConversation(parentId, forkAt, { ownership: { kind: "ownerless" } }),
context,
);
expect(await session.snapshot(Current, child.id, context)).toEqual({ value: "committed" });
});
it("rolls every copied base back when later pre-admission assembly fails", async () => {
const Copied = defineDoc<{ value: string }>({
kind: "fork.rollback.copied",
version: 1,
scope: "conversation",
history: "rewindable",
fork: "asOf",
initial: () => ({ value: "copied" }),
});
let failCheckpoint = true;
const Failure = defineDoc<{ count: number }>({
kind: "fork.rollback.failure",
version: 1,
scope: "session",
initial: () => ({ count: 0 }),
checkpointWhen: () => {
if (failCheckpoint) throw new Error("checkpoint failed");
return false;
},
});
const { session, storage, publications } = openTestSession();
const parentId = await createConversation(session);
let forkAt!: EntryId;
await session.commit(async (tx) => {
forkAt = (await tx.appendEntry(parentId, { kind: "point" })).id;
await tx.doc(Copied, parentId);
await tx.doc(Failure);
}, context);
await flush();
const commits = storage.commits.length;
const published = publications.length;
let childId!: ConversationId;
await expect(
session.commit(async (tx) => {
childId = (await tx.forkConversation(parentId, forkAt, { ownership: { kind: "ownerless" } })).id;
(await tx.doc(Failure)).count = 1;
}, context),
).rejects.toThrow("checkpoint failed");
await flush();
expect(storage.commits).toHaveLength(commits);
expect(publications).toHaveLength(published);
expect(await storage.conversation(childId, context)).toBeUndefined();
failCheckpoint = false;
await session.commit(async (tx) => {
(await tx.doc(Failure)).count = 2;
}, context);
expect(await session.snapshot(Failure, context)).toEqual({ count: 2 });
expect(await session.snapshot(Copied, parentId, context)).toEqual({ value: "copied" });
});
it("rolls back a guaranteed Storage rejection without poisoning the Session", async () => {
const Doc = defineDoc<{ value: string }>({
kind: "fork.storage-rejected",
version: 1,
scope: "conversation",
history: "rewindable",
fork: "asOf",
initial: () => ({ value: "source" }),
});
const { session, storage } = openTestSession();
const parentId = await createConversation(session);
let forkAt!: EntryId;
await session.commit(async (tx) => {
forkAt = (await tx.appendEntry(parentId, { kind: "point" })).id;
await tx.doc(Doc, parentId);
}, context);
let rejectedChildId!: ConversationId;
storage.failNextCommit(new StorageRejected("copy rejected"));
await expect(
session.commit(async (tx) => {
rejectedChildId = (await tx.forkConversation(parentId, forkAt, { ownership: { kind: "ownerless" } })).id;
}, context),
).rejects.toThrow("copy rejected");
expect(await storage.conversation(rejectedChildId, context)).toBeUndefined();
const next = await session.commit((tx) => tx.createConversation({ ownership: { kind: "ownerless" } }), context);
expect(await storage.conversation(next.id, context)).toEqual(next);
});
it("retires a copied document and can recreate the address in the fork transaction", async () => {
const Doc = defineDoc<{ value: string }>({
kind: "fork.retire-recreate",
version: 1,
scope: "conversation",
history: "rewindable",
fork: "asOf",
initial: () => ({ value: "fresh" }),
});
const { session, storage } = openTestSession();
const parentId = await createConversation(session);
let forkAt!: EntryId;
await session.commit(async (tx) => {
forkAt = (await tx.appendEntry(parentId, { kind: "point" })).id;
(await tx.doc(Doc, parentId)).value = "copied";
}, context);
const child = await session.commit(async (tx) => {
const created = await tx.forkConversation(parentId, forkAt, { ownership: { kind: "ownerless" } });
await tx.retireDoc(Doc, created.id);
(await tx.doc(Doc, created.id)).value = "replacement";
return created;
}, context);
const writes = storage.commits.at(-1)!;
expect(writes.filter((write) => write.type === "document.copy")).toHaveLength(1);
expect(writes.filter((write) => write.type === "document.create")).toHaveLength(1);
expect(writes.filter((write) => write.type === "document.retire")).toHaveLength(1);
expect(await session.snapshot(Doc, child.id, context)).toEqual({ value: "replacement" });
});
it("copies only conversation documents, leaving Session and task documents in their original scopes", async () => {
const Copied = defineDoc<{ value: string }>({
kind: "fork.scope.conversation",
version: 1,
scope: "conversation",
history: "latest",
fork: "current",
initial: () => ({ value: "conversation" }),
});
const SessionOnly = defineDoc<{ value: string }>({
kind: "fork.scope.session",
version: 1,
scope: "session",
initial: () => ({ value: "session" }),
});
const TaskOnly = defineDoc<{ value: string }>({
kind: "fork.scope.task",
version: 1,
scope: "task",
initial: () => ({ value: "task" }),
});
const Work = {
definition: {
name: "fork.scope.work",
version: 1,
initial: () => ({ phase: "start" }),
},
};
const { session, storage, publications } = openTestSession();
const parentId = await createConversation(session);
let forkAt!: EntryId;
let taskId!: TaskId;
await session.commit(async (tx) => {
forkAt = (await tx.appendEntry(parentId, { kind: "point" })).id;
taskId = await tx.createTask(Work, null, { conversationId: parentId });
await tx.doc(Copied, parentId);
await tx.doc(SessionOnly);
await tx.doc(TaskOnly, taskId);
}, context);
const child = await session.commit(
(tx) => tx.forkConversation(parentId, forkAt, { ownership: { kind: "ownerless" } }),
context,
);
await flush();
const copies = documentCopies(storage.commits.at(-1)!);
expect(copies.map((write) => write.record.kind)).toEqual([Copied.definition.kind]);
expect(documentCopyChanges(publications.at(-1)!).map((change) => change.record.kind)).toEqual([
Copied.definition.kind,
]);
expect(await session.snapshot(Copied, child.id, context)).toEqual({ value: "conversation" });
expect(await session.snapshot(SessionOnly, context)).toEqual({ value: "session" });
expect(await session.snapshot(TaskOnly, taskId, context)).toEqual({ value: "task" });
});
it("copies every family member across storage scan pages", async () => {
const Family = defineDocFamily<{ value: number }, number>({
kind: "fork.pagination",
version: 1,
family: true,
scope: "conversation",
history: "latest",
fork: "current",
initial: (seed) => ({ value: seed }),
});
const { session, storage } = openTestSession();
const parentId = await createConversation(session);
const forkAt = await session.commit(
async (tx) => (await tx.appendEntry(parentId, { kind: "point" })).id,
context,
);
await session.commit(async (tx) => {
await Promise.all(
Array.from({ length: 260 }, async (_, index) => {
await tx.doc(Family, parentId, `member-${index}`, index);
}),
);
}, context);
const child = await session.commit(
(tx) => tx.forkConversation(parentId, forkAt, { ownership: { kind: "ownerless" } }),
context,
);
expect(documentCopies(storage.commits.at(-1)!)).toHaveLength(260);
expect(await session.snapshot(Family, child.id, "member-0", context)).toEqual({ value: 0 });
expect(await session.snapshot(Family, child.id, "member-259", context)).toEqual({ value: 259 });
});
});

View file

@ -1,7 +1,9 @@
import type { Context } from "@earendil-works/chord";
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
import {
type ConversationId,
type DocumentAddress,
type DocumentId,
type DocumentPoint,
type DocumentRecord,
type Id,
@ -38,6 +40,7 @@ export class ControlledStorage extends MemoryStorage {
/** Detached batches for value assertions. */
readonly commits: (readonly StorageWrite[])[] = [];
mintCount = 0;
documentReadCount = 0;
#commitGate: { gate: Deferred; entered: Deferred } | undefined;
#findGate: { gate: Deferred; entered: Deferred } | undefined;
#commitFailure: Error | undefined;
@ -80,9 +83,14 @@ export class ControlledStorage extends MemoryStorage {
return super.commit(writes, commitContext);
}
override mintId(): Promise<Id> {
override mintId<I extends Id<string>>(): Promise<I> {
this.mintCount++;
return super.mintId();
return super.mintId<I>();
}
override document(id: DocumentId, at: DocumentPoint, callContext: Context) {
this.documentReadCount++;
return super.document(id, at, callContext);
}
override async findDocument(
@ -114,13 +122,26 @@ export function openTestSession(): {
return { storage, session, publications };
}
export function documentChanges(publication: CommitPublication): readonly DocumentCommitChange[] {
return publication.changes.filter((change): change is DocumentCommitChange => change.type === "document");
export function documentChanges(
publication: CommitPublication,
): readonly Extract<DocumentCommitChange, { readonly type: "document" }>[] {
return publication.changes.filter(
(change): change is Extract<DocumentCommitChange, { readonly type: "document" }> => change.type === "document",
);
}
export function documentCopyChanges(
publication: CommitPublication,
): readonly Extract<DocumentCommitChange, { readonly type: "document.copy" }>[] {
return publication.changes.filter(
(change): change is Extract<DocumentCommitChange, { readonly type: "document.copy" }> =>
change.type === "document.copy",
);
}
/** Create one conversation and return its ID. */
export async function createConversation(session: SessionKernel): Promise<Id> {
return session.commit(async (tx) => (await tx.createConversation({})).id, context);
export async function createConversation(session: SessionKernel): Promise<ConversationId> {
return session.commit(async (tx) => (await tx.createConversation({ ownership: { kind: "ownerless" } })).id, context);
}
/** Resolve after pending microtasks and one macrotask turn. */

View file

@ -1,13 +1,16 @@
import type { JsonValue } from "@earendil-works/chord";
import {
type ConversationId,
defineDoc,
defineDocFamily,
type Id,
type EntryId,
ReadAfterWrite,
type Task,
type TaskId,
type TaskRecord,
} from "@earendil-works/pi-durable";
import { describe, expect, it } from "vitest";
import { idFromNumber } from "../src/ids.ts";
import { context, createConversation, documentChanges, flush, openTestSession } from "./session-support.ts";
type Checkpoint = { phase: "start" } | { phase: "next"; step: number };
@ -58,13 +61,13 @@ function terminal(task: AnyTask): AnyTask {
async function createTask(
session: ReturnType<typeof openTestSession>["session"],
conversationId: Id,
conversationId: ConversationId,
withDocument = false,
): Promise<Id> {
): Promise<TaskId<{ ok: boolean }>> {
return session.commit(async (tx) => {
const ref = await tx.createTask(WorkTask, { path: "a" }, { conversationId });
if (withDocument) (await tx.doc(ProgressDoc, ref.id)).lines.push("started");
return ref.id;
const taskId = await tx.createTask(WorkTask, { path: "a" }, { conversationId });
if (withDocument) (await tx.doc(ProgressDoc, taskId)).lines.push("started");
return taskId;
}, context);
}
@ -74,14 +77,16 @@ describe("Session transaction tables", () => {
const conversationId = await createConversation(session);
await session.commit(async (tx) => {
expect(await tx.conversation(conversationId)).toEqual({ id: conversationId });
expect(await tx.scanConversations(1)).toEqual({ items: [{ id: conversationId }] });
expect(await tx.scanConversations({}, 1)).toEqual({ items: [{ id: conversationId }] });
expect(await tx.scanTasks({ conversationId }, 10)).toEqual({ items: [] });
expect(await tx.scanEntries({ conversationId }, 10)).toEqual({ items: [] });
await tx.appendEntry(conversationId, { kind: "note" });
await expect(tx.conversation(conversationId)).rejects.toBeInstanceOf(ReadAfterWrite);
await expect(tx.task(1)).rejects.toThrow("Tx.task() cannot read tables after the first table write");
await expect(tx.entry(1)).rejects.toBeInstanceOf(ReadAfterWrite);
await expect(tx.scanConversations(10)).rejects.toBeInstanceOf(ReadAfterWrite);
await expect(tx.task(idFromNumber<TaskId>(1))).rejects.toThrow(
"Tx.task() cannot read tables after the first table write",
);
await expect(tx.entry(idFromNumber<EntryId>(1))).rejects.toBeInstanceOf(ReadAfterWrite);
await expect(tx.scanConversations({}, 10)).rejects.toBeInstanceOf(ReadAfterWrite);
await expect(tx.scanEntries({ conversationId }, 10)).rejects.toBeInstanceOf(ReadAfterWrite);
// Document access remains available after table writes.
(await tx.doc(NotesDoc, conversationId)).text = "after write";
@ -97,10 +102,10 @@ describe("Session transaction tables", () => {
await createConversation(session),
];
await session.commit(async (tx) => {
const first = await tx.scanConversations(2);
const first = await tx.scanConversations({}, 2);
expect(first.items.map(({ id }) => id)).toEqual(ids.slice(0, 2));
expect(first.next).toBeDefined();
const second = await tx.scanConversations(2, first.next);
const second = await tx.scanConversations({}, 2, first.next);
expect(second.items.map(({ id }) => id)).toEqual(ids.slice(2));
expect(second.next).toBeUndefined();
}, context);
@ -120,7 +125,7 @@ describe("Session transaction tables", () => {
it("creates conversations, entries, and tasks with minted IDs", async () => {
const { session, storage, publications } = openTestSession();
const created = await session.commit(async (tx) => {
const conversation = await tx.createConversation({});
const conversation = await tx.createConversation({ ownership: { kind: "ownerless" } });
const first = await tx.appendEntry(conversation.id, { kind: "note", data: "one" });
const headed = await tx.appendEntry(conversation.id, { kind: "summary", head: "self" });
const task = await tx.createTask(
@ -130,7 +135,7 @@ describe("Session transaction tables", () => {
);
return { conversation, first, headed, task };
}, context);
const ids = [created.conversation.id, created.first.id, created.headed.id, created.task.id];
const ids = [created.conversation.id, created.first.id, created.headed.id, created.task];
expect(new Set(ids).size).toBe(4);
expect(created.headed.head).toBe(created.headed.id);
expect(created.first).toEqual({
@ -140,8 +145,8 @@ describe("Session transaction tables", () => {
data: "one",
});
expect((await storage.entry(created.headed.id, context))!.entry).toEqual(created.headed);
expect(await storage.task(created.task.id, context)).toEqual({
id: created.task.id,
expect(await storage.task(created.task, context)).toEqual({
id: created.task,
conversationId: created.conversation.id,
kind: "test.work",
version: 1,
@ -166,9 +171,115 @@ describe("Session transaction tables", () => {
await expect(session.commit((tx) => tx.createTask(WorkTask, { path: "x" }), context)).rejects.toThrow(
"requires options.conversationId",
);
await expect(session.commit((tx) => tx.appendEntry(12345, { kind: "note" }), context)).rejects.toThrow(
"Conversation 12345 does not exist",
);
await expect(
session.commit((tx) => tx.appendEntry(idFromNumber<ConversationId>(12345), { kind: "note" }), context),
).rejects.toThrow("Conversation 12345 does not exist");
});
it("creates a conversation with an explicitly staged task owner", async () => {
const { session, storage } = openTestSession();
const parentId = await createConversation(session);
const created = await session.commit(async (tx) => {
const supervisorId = await tx.createTask(
WorkTask,
{ path: "background" },
{ conversationId: parentId, background: true },
);
const child = await tx.createConversation({ ownership: { kind: "task", taskId: supervisorId } });
return { supervisorId, child };
}, context);
expect(created.child.owner).toEqual({ conversationId: parentId, taskId: created.supervisorId });
expect(await storage.conversation(created.child.id, context)).toEqual(created.child);
await expect(
session.commit(async (tx) => {
const supervisor = (await tx.task(created.supervisorId))!;
tx.setTask({ ...supervisor, conversationId: created.child.id });
}, context),
).rejects.toThrow(`Task ${created.supervisorId} cannot change conversations`);
expect((await storage.task(created.supervisorId, context))?.conversationId).toBe(parentId);
});
it("rejects missing, terminal, and abort-marked conversation owners atomically", async () => {
const { session, storage } = openTestSession();
const parentId = await createConversation(session);
let movedStagedTaskId: TaskId<{ ok: boolean }> | undefined;
await expect(
session.commit(async (tx) => {
const taskId = await tx.createTask(WorkTask, { path: "move" }, { conversationId: parentId });
movedStagedTaskId = taskId;
tx.setTask({
id: taskId,
conversationId: idFromNumber<ConversationId>(998),
kind: WorkTask.definition.name,
version: WorkTask.definition.version,
input: { path: "move" },
after: [],
background: false,
abortRequested: false,
state: { status: "pending", checkpoint: { phase: "start" } },
});
}, context),
).rejects.toThrow("cannot change conversations");
if (movedStagedTaskId === undefined) throw new Error("Expected a staged task ID");
expect(await storage.task(movedStagedTaskId, context)).toBeUndefined();
const missingTaskId = idFromNumber<TaskId>(999);
await expect(
session.commit((tx) => tx.createConversation({ ownership: { kind: "task", taskId: missingTaskId } }), context),
).rejects.toThrow("Conversation owner task 999 does not exist");
let rejectedChildId: ConversationId | undefined;
await expect(
session.commit(async (tx) => {
const supervisorId = await tx.createTask(WorkTask, { path: "aborting" }, { conversationId: parentId });
rejectedChildId = (await tx.createConversation({ ownership: { kind: "task", taskId: supervisorId } })).id;
tx.setTask({
id: supervisorId,
conversationId: parentId,
kind: WorkTask.definition.name,
version: WorkTask.definition.version,
input: { path: "aborting" },
after: [],
background: false,
abortRequested: true,
state: { status: "pending", checkpoint: { phase: "start" } },
});
}, context),
).rejects.toThrow("is abort-marked");
expect(rejectedChildId).toBeDefined();
if (rejectedChildId === undefined) throw new Error("Expected a rejected child ID");
expect(await storage.conversation(rejectedChildId, context)).toBeUndefined();
await expect(
session.commit(async (tx) => {
const supervisorId = await tx.createTask(WorkTask, { path: "terminal" }, { conversationId: parentId });
await tx.createConversation({ ownership: { kind: "task", taskId: supervisorId } });
tx.setTask({
id: supervisorId,
conversationId: parentId,
kind: WorkTask.definition.name,
version: WorkTask.definition.version,
input: { path: "terminal" },
after: [],
background: false,
abortRequested: false,
state: { status: "terminal", outcome: { status: "completed", result: { ok: true } } },
});
}, context),
).rejects.toThrow("is terminal");
const terminalOwnerId = await createTask(session, parentId);
await session.commit(async (tx) => {
tx.setTask(terminal((await tx.task(terminalOwnerId))!));
}, context);
await expect(
session.commit(
(tx) => tx.createConversation({ ownership: { kind: "task", taskId: terminalOwnerId } }),
context,
),
).rejects.toThrow("is terminal");
});
it("takes ownership of table JSON and rejects non-strict values", async () => {
@ -227,11 +338,11 @@ describe("Session transaction tables", () => {
const { session, publications } = openTestSession();
const conversationId = await createConversation(session);
const taskId = await session.commit(async (tx) => {
const ref = await tx.createTask(WorkTask, { path: "a" }, { conversationId });
const createdTaskId = await tx.createTask(WorkTask, { path: "a" }, { conversationId });
// Validation uses the candidate task record, not a caller table read.
(await tx.doc(ProgressDoc, ref.id)).lines.push("created");
(await tx.doc(StepDoc, ref.id, "one", null)).lines.push("step");
return ref.id;
(await tx.doc(ProgressDoc, createdTaskId)).lines.push("created");
(await tx.doc(StepDoc, createdTaskId, "one", null)).lines.push("step");
return createdTaskId;
}, context);
expect(await session.snapshot(ProgressDoc, taskId, context)).toEqual({ lines: ["created"] });
await flush();
@ -245,7 +356,7 @@ describe("Session transaction tables", () => {
for (const document of documents) expect(document.conversationId).toBe(conversationId);
await session.commit(async (tx) => {
await tx.createConversation({});
await tx.createConversation({ ownership: { kind: "ownerless" } });
(await tx.doc(ProgressDoc, taskId)).lines.push("committed task");
}, context);
await flush();
@ -318,11 +429,11 @@ describe("Session transaction tables", () => {
it("validates document owners", async () => {
const { session } = openTestSession();
await expect(session.commit((tx) => tx.doc(ProgressDoc, 4242).then(() => undefined), context)).rejects.toThrow(
"Task 4242 does not exist",
);
await expect(session.commit((tx) => tx.doc(NotesDoc, 4242).then(() => undefined), context)).rejects.toThrow(
"Conversation 4242 does not exist",
);
await expect(
session.commit((tx) => tx.doc(ProgressDoc, idFromNumber<TaskId>(4242)).then(() => undefined), context),
).rejects.toThrow("Task 4242 does not exist");
await expect(
session.commit((tx) => tx.doc(NotesDoc, idFromNumber<ConversationId>(4242)).then(() => undefined), context),
).rejects.toThrow("Conversation 4242 does not exist");
});
});

View file

@ -1,8 +1,11 @@
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
import { describe, expect, it } from "vitest";
import { StorageRejected } from "../src/errors.ts";
import { idFromNumber } from "../src/ids.ts";
import type { SqliteDatabase, SqliteStatement } from "../src/storage/sqlite/index.ts";
import { SqliteStorage } from "../src/storage/sqlite/index.ts";
import { type NodeSqliteDatabase, openNodeSqliteDatabase } from "../src/storage/sqlite/node.ts";
import { type EntryId, ROOT_CONVERSATION_ID } from "../src/types.ts";
type SettlementMode = "immediate" | "delay" | "reject";
@ -70,24 +73,29 @@ describe("portable SQLite facade settlement", () => {
it("prepares each storage statement once and rebinds it across commits", async () => {
const database = new ControlledSettlementDatabase(await openNodeSqliteDatabase(":memory:"));
const storage = await SqliteStorage.open(database);
await storage.commit([{ type: "conversation", value: { id: 1 } }], BACKGROUND_CONTEXT);
await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT);
await storage.commit(
Array.from({ length: 100 }, (_, index) => ({
type: "entry" as const,
value: { id: index + 2, conversationId: 1, kind: "cached" },
value: { id: idFromNumber<EntryId>(index + 2), conversationId: ROOT_CONVERSATION_ID, kind: "cached" },
})),
BACKGROUND_CONTEXT,
);
await storage.commit(
[{ type: "entry", value: { id: 102, conversationId: 1, kind: "cached-again" } }],
[
{
type: "entry",
value: { id: idFromNumber<EntryId>(102), conversationId: ROOT_CONVERSATION_ID, kind: "cached-again" },
},
],
BACKGROUND_CONTEXT,
);
expect((await storage.entry(2, BACKGROUND_CONTEXT))?.entry.kind).toBe("cached");
expect((await storage.entry(102, BACKGROUND_CONTEXT))?.entry.kind).toBe("cached-again");
await expect(storage.commit([{ type: "conversation", value: { id: 1 } }], BACKGROUND_CONTEXT)).rejects.toThrow(
"ID 1 already belongs to conversation",
);
expect((await storage.entry(2, BACKGROUND_CONTEXT))?.entry.kind).toBe("cached");
expect((await storage.entry(idFromNumber<EntryId>(2), BACKGROUND_CONTEXT))?.entry.kind).toBe("cached");
expect((await storage.entry(idFromNumber<EntryId>(102), BACKGROUND_CONTEXT))?.entry.kind).toBe("cached-again");
await expect(
storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT),
).rejects.toThrow("ID 1 already belongs to conversation");
expect((await storage.entry(idFromNumber<EntryId>(2), BACKGROUND_CONTEXT))?.entry.kind).toBe("cached");
expect(database.prepareCount("SELECT record, commit_seq FROM entries WHERE id = ?")).toBe(1);
expect(database.prepareCount("INSERT OR IGNORE INTO record_ids (id, record_type) VALUES (?, ?)")).toBe(1);
expect(
@ -107,6 +115,20 @@ describe("portable SQLite facade settlement", () => {
database.close();
});
it("does not preserve a guaranteed rejection when rollback itself fails", async () => {
const database = await openNodeSqliteDatabase(":memory:");
database.exec("CREATE TABLE rollback_probe (value INTEGER)");
expect(() =>
database.transaction(() => {
database.exec("INSERT INTO rollback_probe (value) VALUES (1)");
database.exec("COMMIT");
throw new StorageRejected("rejected after an escaped commit");
}),
).toThrow(AggregateError);
expect(database.prepare("SELECT value FROM rollback_probe").get()).toEqual({ value: 1 });
database.close();
});
it("awaits async transaction settlement and adopts IDs only after success", async () => {
const database = new ControlledSettlementDatabase(await openNodeSqliteDatabase(":memory:"));
database.controlNextSettlement("delay");
@ -122,7 +144,12 @@ describe("portable SQLite facade settlement", () => {
database.controlNextSettlement("delay");
const committing = storage.commit(
[{ type: "entry", value: { id: 100, conversationId: 1, kind: "settled" } }],
[
{
type: "entry",
value: { id: idFromNumber<EntryId>(100), conversationId: ROOT_CONVERSATION_ID, kind: "settled" },
},
],
BACKGROUND_CONTEXT,
);
let committed = false;
@ -131,21 +158,26 @@ describe("portable SQLite facade settlement", () => {
});
await Promise.resolve();
expect(committed).toBe(false);
expect(await storage.mintId()).toBe(2);
expect(await storage.mintId<EntryId>()).toBe(2);
database.settle();
await expect(committing).resolves.toBe(1);
expect(await storage.mintId()).toBe(101);
expect(await storage.mintId<EntryId>()).toBe(101);
database.controlNextSettlement("reject");
const rejected = storage.commit(
[{ type: "entry", value: { id: 200, conversationId: 1, kind: "rejected" } }],
[
{
type: "entry",
value: { id: idFromNumber<EntryId>(200), conversationId: ROOT_CONVERSATION_ID, kind: "rejected" },
},
],
BACKGROUND_CONTEXT,
);
expect(await storage.mintId()).toBe(102);
expect(await storage.mintId<EntryId>()).toBe(102);
database.settle();
await expect(rejected).rejects.toThrow("controlled settlement rejection");
expect(await storage.mintId()).toBe(103);
expect(await storage.entry(200, BACKGROUND_CONTEXT)).toBeUndefined();
expect(await storage.mintId<EntryId>()).toBe(103);
expect(await storage.entry(idFromNumber<EntryId>(200), BACKGROUND_CONTEXT)).toBeUndefined();
await storage.close(BACKGROUND_CONTEXT);
});
});

View file

@ -3,6 +3,7 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
import { afterEach, describe, expect, it } from "vitest";
import { idFromNumber } from "../src/ids.ts";
import {
applySqliteMigrations,
CURRENT_SQLITE_SCHEMA_VERSION,
@ -10,7 +11,7 @@ import {
type SqliteMigration,
} from "../src/storage/sqlite/index.ts";
import { openNodeSqliteDatabase, openNodeSqliteStorage } from "../src/storage/sqlite/node.ts";
import { ROOT_CONVERSATION_ID } from "../src/types.ts";
import { type EntryId, ROOT_CONVERSATION_ID } from "../src/types.ts";
const directories = new Set<string>();
@ -102,7 +103,7 @@ describe("durable SQLite migrations", () => {
{
type: "entry",
value: {
id: 2,
id: idFromNumber<EntryId>(2),
conversationId: ROOT_CONVERSATION_ID,
kind: "retained",
data: { retained: true },

View file

@ -6,9 +6,22 @@ import type { Context, JsonValue } from "@earendil-works/chord";
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
import { registerStorageConformance } from "@earendil-works/pi-durable/testing";
import { afterEach, describe, expect, it } from "vitest";
import { idFromNumber } from "../src/ids.ts";
import type { SqliteStorage } from "../src/storage/sqlite/index.ts";
import { type NodeSqliteStorageOptions, openNodeSqliteStorage } from "../src/storage/sqlite/node.ts";
import type { DocumentCreate, EntryRecord, Seq, Storage, StorageWrite, TaskRecord } from "../src/types.ts";
import type {
ConversationId,
DocumentCreate,
DocumentId,
EntryId,
EntryRecord,
Id,
Seq,
Storage,
StorageWrite,
TaskId,
TaskRecord,
} from "../src/types.ts";
import { ROOT_CONVERSATION_ID } from "../src/types.ts";
const context = BACKGROUND_CONTEXT;
@ -53,16 +66,24 @@ class ReopeningStorage implements Storage {
}
}
mintId: Storage["mintId"] = () => this.current.mintId();
mintId<I extends Id<string>>(): Promise<I> {
return this.current.mintId<I>();
}
conversation: Storage["conversation"] = (id, readContext) => this.current.conversation(id, readContext);
scanConversations: Storage["scanConversations"] = (limit, cursor, readContext) =>
this.current.scanConversations(limit, cursor, readContext);
entry(id: number, readContext: Context): ReturnType<Storage["entry"]>;
entry(conversationId: number, id: number, readContext: Context): ReturnType<Storage["entry"]>;
entry(idOrConversationId: number, idOrContext: number | Context, readContext?: Context) {
return readContext === undefined
? this.current.entry(idOrConversationId, idOrContext as Context)
: this.current.entry(idOrConversationId, idOrContext as number, readContext);
scanConversations: Storage["scanConversations"] = (query, limit, cursor, readContext) =>
this.current.scanConversations(query, limit, cursor, readContext);
entry(id: EntryId, readContext: Context): ReturnType<Storage["entry"]>;
entry(conversationId: ConversationId, id: EntryId, readContext: Context): ReturnType<Storage["entry"]>;
entry(idOrConversationId: EntryId | ConversationId, idOrContext: EntryId | Context, readContext?: Context) {
if (readContext === undefined) {
return this.current.entry(idFromNumber<EntryId>(idOrConversationId), idOrContext as Context);
}
if (typeof idOrContext !== "number") throw new TypeError("Storage.entry() requires an entry ID");
return this.current.entry(
idFromNumber<ConversationId>(idOrConversationId),
idFromNumber<EntryId>(idOrContext),
readContext,
);
}
findLatestHeadMarker: Storage["findLatestHeadMarker"] = (conversationId, at, readContext) =>
this.current.findLatestHeadMarker(conversationId, at, readContext);
@ -105,15 +126,15 @@ registerStorageConformance({ describe, expect, it }, "SqliteStorage across reope
}
});
function entry(id: number, conversationId: number, data?: JsonValue): EntryRecord {
function entry(id: EntryId, conversationId: ConversationId, data?: JsonValue): EntryRecord {
return { id, conversationId, kind: "message", ...(data === undefined ? {} : { data }) };
}
async function createRoot(storage: Storage): Promise<number> {
async function createRoot(storage: Storage): Promise<Seq> {
return storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context);
}
function pendingTask(id: number): TaskRecord<JsonValue, JsonValue, JsonValue> {
function pendingTask(id: TaskId<JsonValue>): TaskRecord<JsonValue, JsonValue, JsonValue> {
return {
id,
conversationId: ROOT_CONVERSATION_ID,
@ -132,7 +153,7 @@ function scalar(db: DatabaseSync, sql: string): number {
return row.value;
}
function revisionCount(path: string, documentId: number): number {
function revisionCount(path: string, documentId: DocumentId): number {
const db = new DatabaseSync(path, { readOnly: true });
try {
const row = db
@ -148,7 +169,7 @@ describe("Pico SqliteStorage", () => {
it("persists records, sequence allocation, and global ID allocation across reopen", async () => {
const { storage, path } = await createSqliteStorage();
expect(await createRoot(storage)).toBe(1);
const entryId = await storage.mintId();
const entryId = await storage.mintId<EntryId>();
expect(await storage.commit([{ type: "entry", value: entry(entryId, ROOT_CONVERSATION_ID) }], context)).toBe(2);
await storage.close(context);
openStorages.delete(storage);
@ -159,8 +180,13 @@ describe("Pico SqliteStorage", () => {
entry: entry(entryId, ROOT_CONVERSATION_ID),
commitSeq: 2,
});
expect(await reopened.mintId()).toBe(entryId + 1);
expect(await reopened.commit([{ type: "task", value: pendingTask(await reopened.mintId()) }], context)).toBe(3);
expect(await reopened.mintId<EntryId>()).toBe(entryId + 1);
expect(
await reopened.commit(
[{ type: "task", value: pendingTask(await reopened.mintId<TaskId<JsonValue>>()) }],
context,
),
).toBe(3);
});
it("rejects persisted metadata corruption on reopen", async () => {
@ -179,7 +205,7 @@ describe("Pico SqliteStorage", () => {
it("rejects a document whose required base is missing", async () => {
const { storage, path } = await createSqliteStorage();
await createRoot(storage);
const id = await storage.mintId();
const id = await storage.mintId<DocumentId>();
await storage.commit(
[
{
@ -204,7 +230,7 @@ describe("Pico SqliteStorage", () => {
it("replays detached root replacements and follow-up edits while rejecting corrupt operations", async () => {
const { storage, path } = await createSqliteStorage();
await createRoot(storage);
const id = await storage.mintId();
const id = await storage.mintId<DocumentId>();
await storage.commit(
[
{
@ -280,8 +306,8 @@ describe("Pico SqliteStorage", () => {
it("rolls SQL rows and sequence allocation back as one transaction", async () => {
const { storage, path } = await createSqliteStorage();
await createRoot(storage);
const transientId = await storage.mintId();
const transientTaskId = await storage.mintId();
const transientId = await storage.mintId<EntryId>();
const transientTaskId = await storage.mintId<TaskId<JsonValue>>();
const circular: { self?: unknown } = {};
circular.self = circular;
await expect(
@ -298,7 +324,7 @@ describe("Pico SqliteStorage", () => {
).rejects.toThrow("circular structure");
expect(await storage.entry(transientId, context)).toBeUndefined();
expect(await storage.task(transientTaskId, context)).toBeUndefined();
const committedId = await storage.mintId();
const committedId = await storage.mintId<EntryId>();
expect(await storage.commit([{ type: "entry", value: entry(committedId, ROOT_CONVERSATION_ID) }], context)).toBe(
2,
);
@ -315,7 +341,7 @@ describe("Pico SqliteStorage", () => {
it("reconstructs recent and ancient rewindable points after reopen", async () => {
const { storage, path } = await createSqliteStorage();
await createRoot(storage);
const id = await storage.mintId();
const id = await storage.mintId<DocumentId>();
const record = {
id,
kind: "history",
@ -414,7 +440,7 @@ describe("Pico SqliteStorage", () => {
it("reclaims current-only revisions only after a base or retirement", async () => {
const { storage, path } = await createSqliteStorage();
await createRoot(storage);
const id = await storage.mintId();
const id = await storage.mintId<DocumentId>();
await storage.commit(
[
{
@ -471,7 +497,10 @@ describe("Pico SqliteStorage", () => {
[
{
type: "entry",
value: entry(await storage.mintId(), ROOT_CONVERSATION_ID, { text: "x".repeat(32 * 1024), index }),
value: entry(await storage.mintId<EntryId>(), ROOT_CONVERSATION_ID, {
text: "x".repeat(32 * 1024),
index,
}),
},
],
context,
@ -493,7 +522,7 @@ describe("Pico SqliteStorage", () => {
it("reuses pages released by current-only checkpoints", async () => {
const { storage, path } = await createSqliteStorage();
await createRoot(storage);
const id = await storage.mintId();
const id = await storage.mintId<DocumentId>();
const large = "x".repeat(512 * 1024);
await storage.commit(
[
@ -542,13 +571,16 @@ describe("Pico SqliteStorage", () => {
[
{
type: "entry",
value: entry(await storage.mintId(), ROOT_CONVERSATION_ID, { index, text: "x".repeat(1_024) }),
value: entry(await storage.mintId<EntryId>(), ROOT_CONVERSATION_ID, {
index,
text: "x".repeat(1_024),
}),
},
],
context,
);
}
const documentId = await storage.mintId();
const documentId = await storage.mintId<DocumentId>();
await storage.commit(
[
{

View file

@ -13,7 +13,7 @@ import { afterAll, bench, describe } from "vitest";
import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts";
import { MemoryStorage } from "../src/storage/memory.ts";
import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts";
import type { Storage } from "../src/types.ts";
import type { EntryId, Storage } from "../src/types.ts";
const STORAGE_BENCHMARK_BACKENDS = ["memory", "sqlite", "jsonl"] as const;
type StorageBenchmarkBackend = (typeof STORAGE_BENCHMARK_BACKENDS)[number];
@ -118,7 +118,7 @@ type PersistentBackend = Exclude<StorageBenchmarkBackend, "memory">;
type ReopenFixture = {
readonly backend: PersistentBackend;
readonly path: string;
readonly firstEntryId: number;
readonly firstEntryId: EntryId;
readonly samples: string[];
};
@ -156,7 +156,7 @@ const reopenedStorages: Storage[] = [];
async function reopenAndRead(
backend: PersistentBackend,
path: string,
firstEntryId: number,
firstEntryId: EntryId,
): Promise<{ readonly id: number; readonly storage: Storage }> {
const storage = await openPersistentStorage(backend, path);
const id = (await storage.entry(firstEntryId, BACKGROUND_CONTEXT))?.entry.id ?? -1;

View file

@ -1,20 +1,59 @@
import { expectTypeOf, it } from "vitest";
import { expect, expectTypeOf, it } from "vitest";
import { idFromNumber, seqFromNumber } from "../src/ids.ts";
import type {
ContextEdit,
ConversationId,
DocumentContent,
DocumentCreate,
DocumentId,
DocumentRecord,
EntryId,
StorageWrite,
SubmissionCreate,
SubmissionId,
SubmissionRecord,
TaskId,
TaskOutcome,
TaskRecord,
TaskState,
} from "../src/index.ts";
const conversationId = idFromNumber<ConversationId>(1);
const entryId = idFromNumber<EntryId>(2);
const answerId = idFromNumber<EntryId>(3);
const taskId = idFromNumber<TaskId<number>>(4);
const submissionId = idFromNumber<SubmissionId>(5);
const documentId = idFromNumber<DocumentId>(6);
const seq = seqFromNumber(1);
type TaskResult<I> = I extends TaskId<infer R> ? R : never;
it("brands numeric IDs by record kind and carries task result types", () => {
expect(typeof conversationId).toBe("number");
expect(JSON.stringify(taskId)).toBe("4");
expectTypeOf(conversationId).toMatchTypeOf<number>();
expectTypeOf<TaskResult<typeof taskId>>().toEqualTypeOf<number>();
const compileTimeFailures = () => {
const widenedTask: TaskId = idFromNumber<TaskId<{ ok: boolean }>>(7);
// @ts-expect-error an erased task result cannot be narrowed without a typed source
const narrowedTask: TaskId<string> = widenedTask;
// @ts-expect-error conversation IDs are not task IDs
const wrongTask: TaskId = conversationId;
// @ts-expect-error task IDs are not conversation IDs
const wrongConversation: ConversationId = taskId;
// @ts-expect-error entry IDs are not document IDs
const wrongDocument: DocumentId = entryId;
// @ts-expect-error entity IDs are not commit sequences
const wrongSequence: typeof seq = entryId;
void [narrowedTask, wrongTask, wrongConversation, wrongDocument, wrongSequence];
};
expectTypeOf(compileTimeFailures).toBeFunction();
});
it("encodes discriminator-dependent fields", () => {
const omit = { target: 1, action: "omit" } satisfies ContextEdit;
const replace = { target: 1, action: "replace", messages: [] } satisfies ContextEdit;
const omit = { target: entryId, action: "omit" } satisfies ContextEdit;
const replace = { target: entryId, action: "replace", messages: [] } satisfies ContextEdit;
const pending = { status: "pending", checkpoint: { phase: "ready" } } satisfies TaskState<
{ phase: string },
{ value: number }
@ -24,31 +63,31 @@ it("encodes discriminator-dependent fields", () => {
outcome: { status: "completed", result: { value: 1 } },
} satisfies TaskState<{ phase: string }, { value: number }>;
const completedInput = {
id: 1,
conversationId: 1,
id: submissionId,
conversationId,
type: "input",
status: "done",
entry: 2,
answer: 3,
entry: entryId,
answer: answerId,
} satisfies SubmissionRecord;
const completedWrite = {
id: 4,
conversationId: 1,
id: submissionId,
conversationId,
type: "write",
status: "done",
entry: 5,
entry: entryId,
} satisfies SubmissionRecord;
const queuedWriteCreate = {
conversationId: 1,
conversationId,
type: "write",
status: "queued",
} satisfies SubmissionCreate;
const baseContent = { kind: "base", version: 1, value: { count: 1 } } satisfies DocumentContent;
const deltaContent = { kind: "delta", version: 1, ops: [["s", ["count"], 2]] } satisfies DocumentContent;
const conversationDocument = {
id: 1,
id: documentId,
kind: "test",
scope: { kind: "conversation", conversationId: 1 },
scope: { kind: "conversation", conversationId },
history: "rewindable",
fork: "asOf",
} satisfies DocumentCreate;
@ -57,7 +96,7 @@ it("encodes discriminator-dependent fields", () => {
expectTypeOf(replace.action).toEqualTypeOf<"replace">();
expectTypeOf(pending.status).toEqualTypeOf<"pending">();
expectTypeOf(terminal.status).toEqualTypeOf<"terminal">();
expectTypeOf(completedInput.answer).toEqualTypeOf<number>();
expectTypeOf(completedInput.answer).toEqualTypeOf<EntryId>();
expectTypeOf(completedWrite.type).toEqualTypeOf<"write">();
expectTypeOf(queuedWriteCreate.status).toEqualTypeOf<"queued">();
expectTypeOf(baseContent.kind).toEqualTypeOf<"base">();
@ -66,9 +105,9 @@ it("encodes discriminator-dependent fields", () => {
const compileTimeFailures = () => {
// @ts-expect-error replacement edits require replacement messages
const missingReplacement: ContextEdit = { target: 1, action: "replace" };
const missingReplacement: ContextEdit = { target: entryId, action: "replace" };
// @ts-expect-error omission edits cannot carry replacement messages
const omissionWithMessages: ContextEdit = { target: 1, action: "omit", messages: [] };
const omissionWithMessages: ContextEdit = { target: entryId, action: "omit", messages: [] };
const pendingWithOutcome: TaskState<{ phase: string }, number> = {
status: "pending",
checkpoint: { phase: "ready" },
@ -83,8 +122,8 @@ it("encodes discriminator-dependent fields", () => {
};
// @ts-expect-error terminal task records cannot retain live memos
const terminalWithMemos: TaskRecord<null, { phase: string }, number> = {
id: 1,
conversationId: 1,
id: taskId,
conversationId,
kind: "test",
version: 1,
input: null,
@ -96,50 +135,34 @@ it("encodes discriminator-dependent fields", () => {
};
// @ts-expect-error session documents do not declare conversation history behavior
const sessionWithHistory: DocumentRecord = {
id: 1,
id: documentId,
kind: "test",
createdAt: 1,
createdAt: seq,
scope: { kind: "session" },
history: "latest",
fork: "current",
};
// @ts-expect-error conversation document creation requires history and fork policies
const conversationWithoutPolicy: DocumentCreate = {
id: 1,
id: documentId,
kind: "test",
scope: { kind: "conversation", conversationId: 1 },
};
// @ts-expect-error session document creation cannot declare conversation policies
const sessionCreateWithPolicy: DocumentCreate = {
id: 1,
kind: "test",
scope: { kind: "session" },
history: "latest",
fork: "current",
scope: { kind: "conversation", conversationId },
};
const taskWithPolicy = {
id: 1,
id: documentId,
kind: "test",
scope: { kind: "task", taskId: 1 },
scope: { kind: "task", taskId },
history: "latest",
fork: "initial",
} as const;
// @ts-expect-error task document creation cannot declare conversation policies
const taskCreateWithPolicy: DocumentCreate = taskWithPolicy;
// @ts-expect-error latest document creation cannot use as-of fork behavior
const latestCreateWithAsOf: DocumentCreate = {
id: 1,
kind: "test",
scope: { kind: "conversation", conversationId: 1 },
history: "latest",
fork: "asOf",
};
const createWithSequence: DocumentCreate = {
id: 1,
id: documentId,
kind: "test",
scope: { kind: "session" },
// @ts-expect-error storage, not the create command, supplies createdAt
createdAt: 1,
createdAt: seq,
};
// @ts-expect-error document bases cannot carry operation batches
const baseWithOps: DocumentContent = { kind: "base", version: 1, value: {}, ops: [] };
@ -147,7 +170,7 @@ it("encodes discriminator-dependent fields", () => {
const deltaWithValue: DocumentContent = { kind: "delta", version: 1, ops: [], value: {} };
const createWithDelta: StorageWrite = {
type: "document.create",
record: { id: 1, kind: "test", scope: { kind: "session" } },
record: { id: documentId, kind: "test", scope: { kind: "session" } },
// @ts-expect-error document creation always starts from a complete base
content: { kind: "delta", version: 1, ops: [] },
};
@ -159,50 +182,33 @@ it("encodes discriminator-dependent fields", () => {
};
// @ts-expect-error queued submissions cannot reference transcript entries
const queuedWithEntry: SubmissionRecord = {
id: 1,
conversationId: 1,
id: submissionId,
conversationId,
type: "input",
status: "queued",
entry: 2,
entry: entryId,
};
// @ts-expect-error successful input submissions require an answer entry
const inputWithoutAnswer: SubmissionRecord = {
id: 2,
conversationId: 1,
id: submissionId,
conversationId,
type: "input",
status: "done",
entry: 3,
entry: entryId,
};
// @ts-expect-error passive write submissions never carry an answer
const writeWithAnswer: SubmissionRecord = {
id: 4,
conversationId: 1,
id: submissionId,
conversationId,
type: "write",
status: "done",
entry: 5,
answer: 6,
};
// @ts-expect-error passive writes have no placed intermediate state
const placedWrite: SubmissionRecord = {
id: 7,
conversationId: 1,
type: "write",
status: "placed",
entry: 8,
};
// @ts-expect-error failed passive writes cannot reference an entry
const unansweredWriteWithEntry: SubmissionRecord = {
id: 9,
conversationId: 1,
type: "write",
status: "unanswered",
reason: "failed",
entry: 10,
entry: entryId,
answer: answerId,
};
const submissionCreateWithId: SubmissionCreate = {
// @ts-expect-error Session, not the submission create value, assigns its ID
id: 11,
conversationId: 1,
id: submissionId,
conversationId,
type: "write",
status: "queued",
};
@ -214,9 +220,7 @@ it("encodes discriminator-dependent fields", () => {
terminalWithMemos,
sessionWithHistory,
conversationWithoutPolicy,
sessionCreateWithPolicy,
taskCreateWithPolicy,
latestCreateWithAsOf,
createWithSequence,
baseWithOps,
deltaWithValue,
@ -225,8 +229,6 @@ it("encodes discriminator-dependent fields", () => {
queuedWithEntry,
inputWithoutAnswer,
writeWithAnswer,
placedWrite,
unansweredWriteWithEntry,
submissionCreateWithId,
];
};

View file

@ -106,6 +106,8 @@ const text = styleText(
`Color` is an indexed ANSI color, an sRGB color, or an OKLCH color. Every color converts to sRGB, so color math such as `mixColors()` always works. Indices 0-15 follow the user's terminal palette, so their sRGB values are approximations. `styleText()` converts colors to truecolor or 256-color output based on the requested terminal mode.
`parseColor()` also accepts OKHSL, as in `okhsl(250 60% 55%)`; `okhslColor()` builds it in code and `colorToOkhsl()` reads any color's OKHSL channels. OKHSL saturation is relative to the most the sRGB gamut allows at the hue and lightness, so every value is in gamut and equal saturation looks equally colorful across hues. OKHSL colors are converted to sRGB when created.
Conversions are not cached. OKLCH colors, especially ones outside the sRGB gamut, are more expensive to convert than sRGB or indexed colors. For colors used on every render, convert once and reuse the result:
```typescript

View file

@ -1,3 +1,4 @@
import { linearSrgbToRgb, okhslToRgb, oklabToLinearSrgb, rgbToOkhsl, rgbToOklab } from "./oklab.ts";
import type { RgbColor } from "./terminal-colors.ts";
export interface IndexedColor {
@ -30,6 +31,16 @@ export interface OklchChannels {
h: number;
}
/**
* OKHSL channels: hue in degrees, saturation and lightness 0-1. Saturation is relative to the most the
* sRGB gamut allows at that hue and lightness, so every value is in gamut.
*/
export interface OkhslChannels {
h: number;
s: number;
l: number;
}
export interface TextAttributes {
bold?: boolean;
dim?: boolean;
@ -81,6 +92,31 @@ const OKLCH_PATTERN = new RegExp(
`^oklch\\(\\s*(${NUMBER_PATTERN})(%)?\\s+(${NUMBER_PATTERN})\\s+(${NUMBER_PATTERN})(?:deg)?\\s*\\)$`,
"i",
);
const OKHSL_PATTERN = new RegExp(
`^okhsl\\(\\s*(${NUMBER_PATTERN})(?:deg)?\\s+(${NUMBER_PATTERN})(%)?\\s+(${NUMBER_PATTERN})(%)?\\s*\\)$`,
"i",
);
/**
* An OKHSL color, converted to sRGB. Saturation is relative to the sRGB gamut at the hue and lightness,
* so equal saturation looks equally colorful across hues and lightness.
* @param h Hue in degrees.
* @param s Saturation, 0-1.
* @param l Lightness, 0-1.
*/
export function okhslColor(h: number, s: number, l: number): RgbColorValue {
requireFinite(h, "h");
requireFinite(s, "s");
requireFinite(l, "l");
if (s < 0 || s > 1) throw new Error(`s must be between 0 and 1: ${s}`);
if (l < 0 || l > 1) throw new Error(`l must be between 0 and 1: ${l}`);
const { r, g, b } = okhslToRgb(h, s, l);
return rgbColor(r, g, b);
}
export function colorToOkhsl(color: Color): OkhslChannels {
return rgbToOkhsl(colorToRgb(color));
}
export function parseColor(value: string | number): Color {
if (typeof value === "number") return indexedColor(value);
@ -101,6 +137,13 @@ export function parseColor(value: string | number): Color {
return oklchColor(lightness, Number.parseFloat(oklch[3]), Number.parseFloat(oklch[4]));
}
const okhsl = OKHSL_PATTERN.exec(value);
if (okhsl) {
const saturation = Number.parseFloat(okhsl[2]) / (okhsl[3] ? 100 : 1);
const lightness = Number.parseFloat(okhsl[4]) / (okhsl[5] ? 100 : 1);
return okhslColor(Number.parseFloat(okhsl[1]), saturation, lightness);
}
throw new Error(`Invalid color value: ${value}`);
}
@ -139,65 +182,9 @@ function indexedToRgb(index: number): RgbColor {
return { r: gray, g: gray, b: gray };
}
function srgbToLinear(channel: number): number {
return channel <= 0.04045 ? channel / 12.92 : ((channel + 0.055) / 1.055) ** 2.4;
}
function linearToSrgb(channel: number): number {
return channel <= 0.0031308 ? channel * 12.92 : 1.055 * channel ** (1 / 2.4) - 0.055;
}
interface OklabChannels {
l: number;
a: number;
b: number;
}
interface LinearRgbChannels {
r: number;
g: number;
b: number;
}
function rgbToOklab({ r, g, b }: RgbColor): OklabChannels {
const linearR = srgbToLinear(r / 255);
const linearG = srgbToLinear(g / 255);
const linearB = srgbToLinear(b / 255);
const l = Math.cbrt(0.4122214708 * linearR + 0.5363325363 * linearG + 0.0514459929 * linearB);
const m = Math.cbrt(0.2119034982 * linearR + 0.6806995451 * linearG + 0.1073969566 * linearB);
const s = Math.cbrt(0.0883024619 * linearR + 0.2817188376 * linearG + 0.6299787005 * linearB);
return {
l: 0.2104542553 * l + 0.793617785 * m - 0.0040720468 * s,
a: 1.9779984951 * l - 2.428592205 * m + 0.4505937099 * s,
b: 0.0259040371 * l + 0.7827717662 * m - 0.808675766 * s,
};
}
function oklabToLinearRgb({ l, a, b }: OklabChannels): LinearRgbChannels {
const lRoot = l + 0.3963377774 * a + 0.2158037573 * b;
const mRoot = l - 0.1055613458 * a - 0.0638541728 * b;
const sRoot = l - 0.0894841775 * a - 1.291485548 * b;
const lValue = lRoot ** 3;
const mValue = mRoot ** 3;
const sValue = sRoot ** 3;
return {
r: 4.0767416621 * lValue - 3.3077115913 * mValue + 0.2309699292 * sValue,
g: -1.2684380046 * lValue + 2.6097574011 * mValue - 0.3413193965 * sValue,
b: -0.0041960863 * lValue - 0.7034186147 * mValue + 1.707614701 * sValue,
};
}
function isInSrgbGamut({ r, g, b }: LinearRgbChannels): boolean {
function isInSrgbGamut(linear: number[]): boolean {
const epsilon = 1e-7;
return r >= -epsilon && r <= 1 + epsilon && g >= -epsilon && g <= 1 + epsilon && b >= -epsilon && b <= 1 + epsilon;
}
function linearRgbToChannels({ r, g, b }: LinearRgbChannels): RgbColor {
return {
r: Math.round(Math.max(0, Math.min(1, linearToSrgb(r))) * 255),
g: Math.round(Math.max(0, Math.min(1, linearToSrgb(g))) * 255),
b: Math.round(Math.max(0, Math.min(1, linearToSrgb(b))) * 255),
};
return linear.every((channel) => channel >= -epsilon && channel <= 1 + epsilon);
}
function oklchToRgb({ l, c, h }: OklchChannels): RgbColor {
@ -205,10 +192,10 @@ function oklchToRgb({ l, c, h }: OklchChannels): RgbColor {
const radians = (h * Math.PI) / 180;
const cos = Math.cos(radians);
const sin = Math.sin(radians);
const atChroma = (chroma: number) => oklabToLinearRgb({ l, a: chroma * cos, b: chroma * sin });
const atChroma = (chroma: number) => oklabToLinearSrgb([l, chroma * cos, chroma * sin]);
const direct = atChroma(c);
if (isInSrgbGamut(direct)) return linearRgbToChannels(direct);
if (isInSrgbGamut(direct)) return linearSrgbToRgb(direct);
// Reduce chroma until the color fits. The achromatic color is always in gamut, so it is the
// fallback when no bisection step fits, e.g. `oklch(100% 0.3 150)` must map to white.
@ -225,7 +212,7 @@ function oklchToRgb({ l, c, h }: OklchChannels): RgbColor {
high = chroma;
}
}
return linearRgbToChannels(linear);
return linearSrgbToRgb(linear);
}
export function colorToRgb(color: Color): RgbColor {
@ -241,12 +228,8 @@ export function colorToRgb(color: Color): RgbColor {
export function colorToOklch(color: Color): OklchChannels {
if (color.kind === "oklch") return { l: color.l, c: color.c, h: color.h };
const lab = rgbToOklab(colorToRgb(color));
return {
l: lab.l,
c: Math.hypot(lab.a, lab.b),
h: ((Math.atan2(lab.b, lab.a) * 180) / Math.PI + 360) % 360,
};
const [l, a, b] = rgbToOklab(colorToRgb(color));
return { l, c: Math.hypot(a, b), h: ((Math.atan2(b, a) * 180) / Math.PI + 360) % 360 };
}
export function colorToHex(color: Color): string {

View file

@ -15,14 +15,17 @@ export {
type Color,
type ColorMixSpace,
colorToHex,
colorToOkhsl,
colorToOklch,
colorToRgb,
foregroundAnsi,
type IndexedColor,
indexedColor,
mixColors,
type OkhslChannels,
type OklchChannels,
type OklchColorValue,
okhslColor,
oklchColor,
parseColor,
type RgbColorValue,
@ -101,16 +104,17 @@ export {
export { type RenderLatexOptions, renderLatex } from "./latex.ts";
// Native platform integration
export { getNativeClipboard, type NativeClipboard } from "./native-platform.ts";
export { oklabToOkhslLightness } from "./oklab.ts";
// Input buffering for batch splitting
export { StdinBuffer, type StdinBufferEventMap, type StdinBufferOptions } from "./stdin-buffer.ts";
// Terminal interface and implementations
export { ProcessTerminal, type Terminal } from "./terminal.ts";
// Terminal colors
export {
parseOsc11BackgroundColor,
parseTerminalColorSchemeReport,
type RgbColor,
type TerminalColorScheme,
type TerminalColors,
} from "./terminal-colors.ts";
// Terminal image support
export {

233
packages/tui/src/oklab.ts Normal file
View file

@ -0,0 +1,233 @@
/**
* Oklab and OKHSL <-> sRGB conversion. `colors.ts` builds its OKLCH, OKHSL, and color mixing on it.
*
* Oklab and OKHSL are Björn Ottosson's color spaces; OKHSL's saturation is relative to the sRGB gamut at
* each hue and lightness. This is a port of his reference implementation (https://bottosson.github.io/posts/colorpicker/),
* Copyright (c) 2021 Björn Ottosson, used under the MIT license:
*
* Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
* associated documentation files (the "Software"), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the
* following conditions: The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
* KIND, EXPRESS OR IMPLIED.
*/
import type { RgbColor } from "./terminal-colors.ts";
type Vector = [number, number, number];
type Matrix = [Vector, Vector, Vector];
const multiply = (m: Matrix, [x, y, z]: Vector): Vector =>
m.map((row) => row[0] * x + row[1] * y + row[2] * z) as Vector;
// ============================================================================
// OKHSL <-> sRGB
// ============================================================================
const LINEAR_SRGB_TO_LMS: Matrix = [
[0.4122214694707629, 0.5363325372617349, 0.0514459932675022],
[0.2119034958178251, 0.6806995506452344, 0.1073969535369405],
[0.0883024591900564, 0.2817188391361215, 0.6299787016738222],
];
const LMS_TO_LAB: Matrix = [
[0.210454268309314, 0.793617774702305, -0.0040720430116193],
[1.9779985324311684, -2.42859224204858, 0.450593709617411],
[0.0259040424655478, 0.7827717124575296, -0.8086757549230774],
];
const LAB_TO_LMS: Matrix = [
[1, 0.3963377773761749, 0.2158037573099136],
[1, -0.1055613458156586, -0.0638541728258133],
[1, -0.0894841775298119, -1.2914855480194092],
];
const LMS_TO_LINEAR_SRGB: Matrix = [
[4.0767416360759583, -3.3077115392580629, 0.2309699031821043],
[-1.2684379732850315, 2.6097573492876882, -0.341319376002657],
[-0.0041960761386756, -0.7034186179359362, 1.7076146940746117],
];
/**
* Per sRGB channel (red, green, blue): the (a, b) half-plane where that channel clips
* first, and the polynomial approximating the maximum saturation there.
*/
const SATURATION_FIT: [[number, number], number[]][] = [
[
[-1.8817031, -0.80936501],
[1.19086277, 1.76576728, 0.59662641, 0.75515197, 0.56771245],
],
[
[1.8144408, -1.19445267],
[0.73956515, -0.45954404, 0.08285427, 0.12541073, -0.14503204],
],
[
[0.13110758, 1.81333971],
[1.35733652, -0.00915799, -1.1513021, -0.50559606, 0.00692167],
],
];
const K1 = 0.206;
const K2 = 0.03;
const K3 = (1 + K1) / (1 + K2);
/** Oklab lightness to OKHSL lightness. */
export const oklabToOkhslLightness = (x: number): number =>
0.5 * (K3 * x - K1 + Math.sqrt((K3 * x - K1) ** 2 + 4 * K2 * K3 * x));
/** OKHSL lightness to Oklab lightness. */
const okhslToOklabLightness = (x: number): number => (x * x + K1 * x) / (K3 * (x + K2));
/** sRGB transfer function: linear to encoded channel, both 0-1. */
const linearToSrgb = (value: number): number =>
value > 0.0031308 ? 1.055 * value ** (1 / 2.4) - 0.055 : 12.92 * value;
/** Inverse sRGB transfer function: encoded to linear channel, both 0-1. */
const srgbToLinear = (value: number): number => (value <= 0.04045 ? value / 12.92 : ((value + 0.055) / 1.055) ** 2.4);
/** Oklab [L, a, b] to linear sRGB [r, g, b] (0-1, may leave the gamut). */
export function oklabToLinearSrgb(lab: Vector): Vector {
return multiply(LMS_TO_LINEAR_SRGB, multiply(LAB_TO_LMS, lab).map((value) => value ** 3) as Vector);
}
/** Linear sRGB [r, g, b] (0-1) to Oklab [L, a, b]. */
function linearSrgbToOklab(rgb: Vector): Vector {
return multiply(LMS_TO_LAB, multiply(LINEAR_SRGB_TO_LMS, rgb).map(Math.cbrt) as Vector);
}
/** sRGB channels (0-255) to Oklab [L, a, b]. */
export function rgbToOklab({ r, g, b }: RgbColor): Vector {
return linearSrgbToOklab([r / 255, g / 255, b / 255].map(srgbToLinear) as Vector);
}
/** Linear sRGB [r, g, b] to sRGB channels (0-255, rounded), clipping out-of-gamut channels. */
export function linearSrgbToRgb(linear: Vector): RgbColor {
const [r, g, b] = linear.map((value) => Math.round(Math.min(1, Math.max(0, linearToSrgb(value))) * 255));
return { r, g, b };
}
/** Rate of change of each cube-root LMS component along a chroma direction (a, b). */
function lmsSlopes(a: number, b: number): Vector {
return [LAB_TO_LMS[0], LAB_TO_LMS[1], LAB_TO_LMS[2]].map((row) => row[1] * a + row[2] * b) as Vector;
}
/** Largest saturation (C/L) inside sRGB for hue (a, b): polynomial fit plus one Halley step. */
function maxSaturation(a: number, b: number): number {
const channel = SATURATION_FIT.findIndex(([[x, y]], index) => index === 2 || x * a + y * b > 1);
const [k0, k1, k2, k3, k4] = SATURATION_FIT[channel][1];
const weights = LMS_TO_LINEAR_SRGB[channel];
const saturation = k0 + k1 * a + k2 * b + k3 * a * a + k4 * a * b;
const slopes = lmsSlopes(a, b);
const base = slopes.map((k) => 1 + saturation * k);
const dot = (values: number[]) => values.reduce((sum, value, index) => sum + weights[index] * value, 0);
const f = dot(base.map((value) => value ** 3));
const f1 = dot(base.map((value, index) => 3 * slopes[index] * value ** 2));
const f2 = dot(base.map((value, index) => 6 * slopes[index] ** 2 * value));
return saturation - (f * f1) / (f1 * f1 - 0.5 * f * f2);
}
/** Oklab lightness and chroma of the most saturated sRGB color of hue (a, b). */
function cusp(a: number, b: number): [number, number] {
const saturation = maxSaturation(a, b);
const lightness = Math.cbrt(1 / Math.max(...oklabToLinearSrgb([1, saturation * a, saturation * b])));
return [lightness, lightness * saturation];
}
/** Chroma where the constant-lightness line at `lightness` leaves the sRGB gamut. */
function maxChroma(a: number, b: number, lightness: number, [cuspL, cuspC]: [number, number]): number {
if (lightness <= cuspL) return (cuspC * lightness) / cuspL;
// Upper half: triangle edge, then one Halley step against each channel reaching 1.
const t = (cuspC * (lightness - 1)) / (cuspL - 1);
const slopes = lmsSlopes(a, b);
const lms = slopes.map((k) => lightness + t * k);
const cubes = lms.map((value) => value ** 3);
const first = lms.map((value, index) => 3 * slopes[index] * value ** 2);
const second = lms.map((value, index) => 6 * slopes[index] ** 2 * value);
const dot = (row: Vector, values: number[]) => row[0] * values[0] + row[1] * values[1] + row[2] * values[2];
const steps = LMS_TO_LINEAR_SRGB.map((row) => {
const f = dot(row, cubes) - 1;
const f1 = dot(row, first);
const f2 = dot(row, second);
const u = f1 / (f1 * f1 - 0.5 * f * f2);
return u >= 0 ? -f * u : Number.MAX_VALUE;
});
return t + Math.min(...steps);
}
/** OKHSL's chroma reference points at lightness L and hue (a, b): [c0, cMid, cMax]. */
function chromaStops(L: number, a: number, b: number): [number, number, number] {
const peak = cusp(a, b);
const cMax = maxChroma(a, b, L, peak);
const k = cMax / Math.min(L * (peak[1] / peak[0]), (1 - L) * (peak[1] / (1 - peak[0])));
const midS =
0.11516993 +
1 /
(7.4477897 +
4.1590124 * b +
a *
(-2.19557347 +
1.75198401 * b +
a * (-2.13704948 - 10.02301043 * b + a * (-4.24894561 + 5.38770819 * b + 4.69891013 * a))));
const midT =
0.11239642 +
1 /
(1.6132032 -
0.68124379 * b +
a *
(0.40370612 +
0.90148123 * b +
a * (-0.27087943 + 0.6122399 * b + a * (0.00299215 - 0.45399568 * b - 0.14661872 * a))));
const cMid = 0.9 * k * Math.sqrt(Math.sqrt(1 / (1 / (L * midS) ** 4 + 1 / ((1 - L) * midT) ** 4)));
const c0 = Math.sqrt(1 / (1 / (L * 0.4) ** 2 + 1 / ((1 - L) * 0.8) ** 2));
return [c0, cMid, cMax];
}
/**
* Convert OKHSL to sRGB channels (0-255, rounded), clipping out-of-gamut channels.
* @param hue Hue in degrees.
* @param saturation Saturation, 0-1.
* @param lightness Lightness, 0-1.
*/
export function okhslToRgb(hue: number, saturation: number, lightness: number): RgbColor {
const L = okhslToOklabLightness(lightness);
let lab: Vector = [L, 0, 0];
if (L > 0 && L < 1 && saturation > 0) {
const angle = (2 * Math.PI * (((hue % 360) + 360) % 360)) / 360;
const a = Math.cos(angle);
const b = Math.sin(angle);
const [c0, cMid, cMax] = chromaStops(L, a, b);
// Chroma rises from 0 through cMid at s = 0.8 to cMax at s = 1.
let chroma: number;
if (saturation < 0.8) {
const t = 1.25 * saturation;
const k1 = 0.8 * c0;
chroma = (t * k1) / (1 - (1 - k1 / cMid) * t);
} else {
const t = 5 * (saturation - 0.8);
const k1 = (0.2 * cMid ** 2 * 1.25 ** 2) / c0;
chroma = cMid + (t * k1) / (1 - (1 - k1 / (cMax - cMid)) * t);
}
lab = [L, chroma * a, chroma * b];
}
return linearSrgbToRgb(oklabToLinearSrgb(lab));
}
/**
* Convert sRGB channels (0-255) to OKHSL.
* @returns Hue `h` in degrees (0 for grays), saturation `s` and lightness `l` 0-1.
*/
export function rgbToOkhsl(rgb: RgbColor): { h: number; s: number; l: number } {
const [L, labA, labB] = rgbToOklab(rgb);
const chroma = Math.hypot(labA, labB);
const lightness = oklabToOkhslLightness(L);
if (chroma < 1e-9 || lightness <= 0 || lightness >= 1) return { h: 0, s: 0, l: lightness };
const hue = ((Math.atan2(labB, labA) * 180) / Math.PI + 360) % 360;
const [c0, cMid, cMax] = chromaStops(L, labA / chroma, labB / chroma);
let saturation: number;
if (chroma < cMid) {
const k1 = 0.8 * c0;
saturation = 0.8 * (chroma / (k1 + (1 - k1 / cMid) * chroma));
} else {
const k1 = (0.2 * cMid ** 2 * 1.25 ** 2) / c0;
const offset = chroma - cMid;
saturation = 0.8 + 0.2 * (offset / (k1 + (1 - k1 / (cMax - cMid)) * offset));
}
return { h: hue, s: Math.min(1, Math.max(0, saturation)), l: lightness };
}

View file

@ -6,6 +6,16 @@ export interface RgbColor {
export type TerminalColorScheme = "dark" | "light";
/** Colors the terminal reports for its current theme. */
export interface TerminalColors {
/** Default foreground (OSC 10). */
foreground?: RgbColor;
/** Default background (OSC 11). */
background?: RgbColor;
/** ANSI colors 0-15 (OSC 4). Only set when the terminal reported all 16. */
palette?: RgbColor[];
}
function hexToRgb(hex: string): RgbColor {
const normalized = hex.startsWith("#") ? hex.slice(1) : hex;
const r = parseInt(normalized.slice(0, 2), 16);
@ -25,27 +35,24 @@ function parseOscHexChannel(channel: string): number | undefined {
return Math.round((parseInt(channel, 16) / max) * 255);
}
/** OSC color slots: 10 is the terminal's default foreground, 11 its default background. */
export type OscColorSlot = 10 | 11;
/** What an OSC color reply reports: the default foreground (OSC 10), background (OSC 11), or a palette index (OSC 4). */
export type OscColorTarget = "foreground" | "background" | number;
const OSC_COLOR_RESPONSE_PATTERN = /^\x1b\](1[01]);([^\x07\x1b]*)(?:\x07|\x1b\\)$/i;
const OSC_COLOR_RESPONSE_PATTERN = /^\x1b\](?:(1[01])|4;(\d{1,3}));([^\x07\x1b]*)(?:\x07|\x1b\\)$/i;
const COLOR_SCHEME_REPORT_PATTERN = /^(?:\x1b\[\?997;(1|2)n)+$/;
/**
* Parse an OSC 10/11 color reply. Returns undefined when `data` is not such a reply;
* Parse an OSC 10, 11, or 4 color reply. Returns undefined when `data` is not such a reply;
* `rgb` is undefined when it is a reply with an unparseable color.
*/
export function parseOscColorResponse(data: string): { slot: OscColorSlot; rgb: RgbColor | undefined } | undefined {
export function parseOscColorResponse(data: string): { target: OscColorTarget; rgb: RgbColor | undefined } | undefined {
const match = data.match(OSC_COLOR_RESPONSE_PATTERN);
if (!match) {
return undefined;
}
return { slot: match[1] === "10" ? 10 : 11, rgb: parseOscColorValue(match[2]) };
}
export function parseOsc11BackgroundColor(data: string): RgbColor | undefined {
const response = parseOscColorResponse(data);
return response?.slot === 11 ? response.rgb : undefined;
const target: OscColorTarget =
match[1] === "10" ? "foreground" : match[1] === "11" ? "background" : Number.parseInt(match[2], 10);
return { target, rgb: parseOscColorValue(match[3]) };
}
function parseOscColorValue(rawValue: string): RgbColor | undefined {

View file

@ -141,6 +141,8 @@ export class ProcessTerminal implements Terminal {
private _kittyProtocolActive = false;
private _modifyOtherKeysActive = false;
private keyboardProtocolPushed = false;
/** DA1 replies owed to keyboard protocol queries. Later DA1 replies answer other queries and are forwarded. */
private pendingKeyboardProtocolDeviceAttributes = 0;
private keyboardProtocolNegotiationBuffer = "";
private keyboardProtocolBufferFlushTimer?: ReturnType<typeof setTimeout>;
private stdinBuffer?: StdinBuffer;
@ -215,16 +217,16 @@ export class ProcessTerminal implements Terminal {
// Forward individual sequences to the input handler
this.stdinBuffer.on("data", (sequence) => {
const negotiationSequence = this.readKeyboardProtocolNegotiationSequence(sequence);
if (negotiationSequence === "pending") {
const negotiation = this.readKeyboardProtocolNegotiationSequence(sequence);
if (negotiation === "pending") {
this.scheduleKeyboardProtocolNegotiationBufferFlush();
return; // Wait briefly for the rest of a split Kitty response.
}
if (this.handleKeyboardProtocolNegotiationSequence(negotiationSequence)) {
if (negotiation && this.handleKeyboardProtocolNegotiationSequence(negotiation.parsed)) {
return;
}
this.forwardInputSequence(sequence);
this.forwardInputSequence(negotiation?.sequence ?? sequence);
});
// Re-wrap paste content with bracketed paste markers for existing editor handling
@ -257,15 +259,19 @@ export class ProcessTerminal implements Terminal {
this.setupStdinBuffer();
process.stdin.on("data", this.stdinDataHandler!);
this.keyboardProtocolPushed = true;
this.pendingKeyboardProtocolDeviceAttributes += 1;
this.clearKeyboardProtocolNegotiationBuffer();
process.stdout.write(KITTY_KEYBOARD_PROTOCOL_QUERY);
}
private handleKeyboardProtocolNegotiationSequence(
negotiationSequence: KeyboardProtocolNegotiationSequence | undefined,
negotiationSequence: KeyboardProtocolNegotiationSequence,
): boolean {
if (!negotiationSequence) return false;
this.clearKeyboardProtocolNegotiationBuffer();
if (negotiationSequence.type === "device-attributes") {
if (this.pendingKeyboardProtocolDeviceAttributes === 0) return false;
this.pendingKeyboardProtocolDeviceAttributes -= 1;
}
if (negotiationSequence.type === "kitty-flags") {
if (negotiationSequence.flags !== 0) {
this.disableModifyOtherKeys();
@ -285,15 +291,16 @@ export class ProcessTerminal implements Terminal {
return true;
}
/** Returns the parsed negotiation reply with its full (possibly reassembled) sequence. */
private readKeyboardProtocolNegotiationSequence(
sequence: string,
): KeyboardProtocolNegotiationSequence | "pending" | undefined {
): { parsed: KeyboardProtocolNegotiationSequence; sequence: string } | "pending" | undefined {
if (this.keyboardProtocolNegotiationBuffer) {
const bufferedSequence = this.keyboardProtocolNegotiationBuffer + sequence;
const negotiationSequence = parseKeyboardProtocolNegotiationSequence(bufferedSequence);
if (negotiationSequence) {
this.clearKeyboardProtocolNegotiationBuffer();
return negotiationSequence;
return { parsed: negotiationSequence, sequence: bufferedSequence };
}
if (isKeyboardProtocolNegotiationSequencePrefix(bufferedSequence)) {
this.setKeyboardProtocolNegotiationBuffer(bufferedSequence);
@ -303,7 +310,7 @@ export class ProcessTerminal implements Terminal {
}
const negotiationSequence = parseKeyboardProtocolNegotiationSequence(sequence);
if (negotiationSequence) return negotiationSequence;
if (negotiationSequence) return { parsed: negotiationSequence, sequence };
if (isKeyboardProtocolNegotiationSequencePrefix(sequence)) {
this.setKeyboardProtocolNegotiationBuffer(sequence);
return "pending";

View file

@ -6,11 +6,11 @@ import { performance } from "node:perf_hooks";
import { isKeyRelease, matchesKey } from "./keys.ts";
import type { Terminal } from "./terminal.ts";
import {
type OscColorSlot,
parseOscColorResponse,
parseTerminalColorSchemeReport,
type RgbColor,
type TerminalColorScheme,
type TerminalColors,
} from "./terminal-colors.ts";
import { getCapabilities, isImageLine, setCellDimensions } from "./terminal-image.ts";
import { extractSegments, normalizeTerminalOutput, sliceByColumn, sliceWithWidth, visibleWidth } from "./utils.ts";
@ -81,7 +81,13 @@ export interface TuiMouseDispatchResult extends TuiMouseEventResult {
export function dispatchMouseEvent(component: Component, event: TuiMouseEvent): TuiMouseDispatchResult | undefined {
const result = component.handleMouse?.(event);
if (!result) return undefined;
if ("target" in result) return result as TuiMouseDispatchResult;
if ("target" in result) {
// The component forwarded the event to a child it hosts. Like a delegating container, it routes
// keys to that child itself, so it keeps keyboard focus. Focusing the child directly would leave
// focus on a detached component once the host removes it, e.g. a closed settings submenu.
const forwarded = result as TuiMouseDispatchResult;
return forwarded.focus && component.handleInput ? { ...forwarded, focusTarget: component } : forwarded;
}
if (!result.handled && !result.capture && !result.focus) return undefined;
return {
...result,
@ -137,12 +143,34 @@ export interface Component {
export type TuiInputListenerResult = { consume?: boolean; data?: string } | undefined;
export type TuiInputListener = (data: string) => TuiInputListenerResult;
type PendingOscColorQuery = {
settled: boolean;
resolve: ((rgb: RgbColor | undefined) => void) | undefined;
type PendingTerminalColorQuery = {
foreground?: RgbColor;
background?: RgbColor;
palette: Array<RgbColor | undefined>;
/** Targets that already replied, so duplicates do not count twice. */
replied: Set<string>;
/**
* Receives the result: the promise's resolve until the timeout, then `onLateReply`. Unset once the
* query completed (on the DA1 reply or once every color replied); later replies are ignored.
*/
deliver: ((colors: TerminalColors) => void) | undefined;
timer: NodeJS.Timeout | undefined;
};
const TERMINAL_PALETTE_SIZE = 16;
/** OSC 10 and 11 plus OSC 4 for every palette color. */
const TERMINAL_COLOR_REPLY_COUNT = 2 + TERMINAL_PALETTE_SIZE;
/**
* Default colors, palette colors 0-15, and a trailing primary device attributes (DA1) request.
* Every terminal answers DA1 and terminals answer in order, so the DA1 reply marks the end of
* the color replies, including for terminals that ignore the color queries.
*/
const TERMINAL_COLOR_QUERY = `\x1b]10;?\x07\x1b]11;?\x07${Array.from(
{ length: TERMINAL_PALETTE_SIZE },
(_, index) => `\x1b]4;${index};?\x07`,
).join("")}\x1b[c`;
const DEVICE_ATTRIBUTES_RESPONSE_PATTERN = /^\x1b\[\?[\d;]*c$/;
/**
* Interface for components that can receive focus and display a hardware cursor.
* When focused, the component should emit CURSOR_MARKER at the cursor position
@ -447,9 +475,10 @@ export interface TUI extends Component {
removeInputListener(listener: TuiInputListener): void;
onTerminalColorSchemeChange(listener: (scheme: TerminalColorScheme) => void): () => void;
setTerminalColorSchemeNotifications(enabled: boolean): void;
queryTerminalBackgroundColor(options: { timeoutMs: number }): Promise<RgbColor | undefined>;
queryTerminalForegroundColor(options: { timeoutMs: number }): Promise<RgbColor | undefined>;
queryTerminalColorScheme(options: { timeoutMs: number }): Promise<TerminalColorScheme | undefined>;
queryTerminalColors(options: {
timeoutMs: number;
onLateReply?: (colors: TerminalColors) => void;
}): Promise<TerminalColors>;
}
export const VIEWPORT_TUI = Symbol.for("@earendil-works/pi-tui/viewport");
@ -480,9 +509,11 @@ export abstract class TuiBase extends Container implements TUI {
private clearOnShrink = false;
protected fullRedrawCount = 0;
protected stopped = false;
// Replies are counted separately from queries so late replies after a timeout are still consumed.
private pendingOscColorReplies: Record<OscColorSlot, number> = { 10: 0, 11: 0 };
private pendingOscColorQueries: Record<OscColorSlot, PendingOscColorQuery[]> = { 10: [], 11: [] };
/**
* Color queries waiting for their DA1 reply, oldest first. Terminals answer in order, so color
* replies belong to the oldest one. Queries stay here after a timeout to collect late replies.
*/
private pendingTerminalColorQueries: PendingTerminalColorQuery[] = [];
private terminalColorSchemeListeners = new Set<(scheme: TerminalColorScheme) => void>();
private terminalColorSchemeNotificationsEnabled = false;
/** Directory for debug/crash logs. When undefined, debug logging is disabled and crash dumps fall back to the OS temp directory. */
@ -1011,7 +1042,7 @@ export abstract class TuiBase extends Container implements TUI {
}
private handleTerminalInput(data: string): void {
if (this.consumeOscColorResponse(data)) {
if (this.consumeTerminalColorResponse(data)) {
return;
}
if (this.consumeTerminalColorSchemeReport(data)) {
@ -1088,27 +1119,52 @@ export abstract class TuiBase extends Container implements TUI {
}
}
private consumeOscColorResponse(data: string): boolean {
const response = parseOscColorResponse(data);
if (!response || this.pendingOscColorReplies[response.slot] <= 0) {
private consumeTerminalColorResponse(data: string): boolean {
const query = this.pendingTerminalColorQueries[0];
if (!query) {
return false;
}
if (DEVICE_ATTRIBUTES_RESPONSE_PATTERN.test(data)) {
this.pendingTerminalColorQueries.shift();
this.completeTerminalColorQuery(query);
return true;
}
const { slot, rgb } = response;
this.pendingOscColorReplies[slot] -= 1;
const query = this.pendingOscColorQueries[slot].shift();
if (query && !query.settled) {
query.settled = true;
if (query.timer) {
clearTimeout(query.timer);
query.timer = undefined;
}
query.resolve?.(rgb);
query.resolve = undefined;
const response = parseOscColorResponse(data);
if (!response) {
return false;
}
const { target, rgb } = response;
const key = String(target);
if (!query.deliver || query.replied.has(key)) {
return true;
}
query.replied.add(key);
if (target === "foreground") {
query.foreground = rgb;
} else if (target === "background") {
query.background = rgb;
} else if (target < TERMINAL_PALETTE_SIZE) {
query.palette[target] = rgb;
}
if (query.replied.size === TERMINAL_COLOR_REPLY_COUNT) {
this.completeTerminalColorQuery(query);
}
return true;
}
private terminalColorQueryResult(query: PendingTerminalColorQuery): TerminalColors {
const palette = query.palette.every((color) => color !== undefined) ? (query.palette as RgbColor[]) : undefined;
return { foreground: query.foreground, background: query.background, palette };
}
private completeTerminalColorQuery(query: PendingTerminalColorQuery): void {
const deliver = query.deliver;
query.deliver = undefined;
clearTimeout(query.timer);
deliver?.(this.terminalColorQueryResult(query));
}
private consumeTerminalColorSchemeReport(data: string): boolean {
const scheme = parseTerminalColorSchemeReport(data);
if (!scheme) {
@ -1404,70 +1460,34 @@ export abstract class TuiBase extends Container implements TUI {
}
/**
* Query the terminal's default background color with OSC 11 (`ESC ] 11 ; ? BEL`).
* @param timeoutMs Query timeout in milliseconds.
* @returns Promise containing the parsed RGB color, or undefined if it times out or fails to parse.
* Query the terminal's theme colors: the default foreground (OSC 10), the default background
* (OSC 11), and ANSI colors 0-15 (OSC 4), followed by a DA1 request that marks the end of the
* replies. Resolves when the DA1 reply or all color replies arrive, or when the timeout expires.
* Colors the terminal did not report are undefined; the palette is only set when all 16 arrived.
* @param timeoutMs Query timeout in milliseconds, for terminals that do not answer DA1 either.
* @param onLateReply Receives the replies if the query completes after the timeout, e.g. over slow links.
*/
queryTerminalBackgroundColor({ timeoutMs }: { timeoutMs: number }): Promise<RgbColor | undefined> {
return this.queryTerminalColor(11, timeoutMs);
}
/**
* Query the terminal's default foreground color with OSC 10 (`ESC ] 10 ; ? BEL`).
* @param timeoutMs Query timeout in milliseconds.
* @returns Promise containing the parsed RGB color, or undefined if it times out or fails to parse.
*/
queryTerminalForegroundColor({ timeoutMs }: { timeoutMs: number }): Promise<RgbColor | undefined> {
return this.queryTerminalColor(10, timeoutMs);
}
private queryTerminalColor(slot: OscColorSlot, timeoutMs: number): Promise<RgbColor | undefined> {
queryTerminalColors({
timeoutMs,
onLateReply,
}: {
timeoutMs: number;
onLateReply?: (colors: TerminalColors) => void;
}): Promise<TerminalColors> {
return new Promise((resolve) => {
const query: PendingOscColorQuery = {
settled: false,
resolve,
const query: PendingTerminalColorQuery = {
palette: Array.from({ length: TERMINAL_PALETTE_SIZE }, () => undefined),
replied: new Set(),
deliver: resolve,
timer: undefined,
};
// Resolve with the replies so far, and keep collecting late replies for `onLateReply`.
query.timer = setTimeout(() => {
if (query.settled) {
return;
}
query.settled = true;
query.timer = undefined;
query.resolve?.(undefined);
query.resolve = undefined;
query.deliver = onLateReply;
resolve(this.terminalColorQueryResult(query));
}, timeoutMs);
this.pendingOscColorQueries[slot].push(query);
this.pendingOscColorReplies[slot] += 1;
this.terminal.write(`\x1b]${slot};?\x07`);
});
}
/**
* Query the terminal's color-scheme preference with DSR (`CSI ? 996 n`).
* Terminals that support the color palette notification protocol reply with
* `CSI ? 997 ; 1 n` for dark or `CSI ? 997 ; 2 n` for light.
*/
queryTerminalColorScheme({ timeoutMs }: { timeoutMs: number }): Promise<TerminalColorScheme | undefined> {
return new Promise((resolve) => {
let settled = false;
let timer: NodeJS.Timeout | undefined;
let unsubscribe: () => void = () => {};
const settle = (scheme: TerminalColorScheme | undefined) => {
if (settled) return;
settled = true;
if (timer) {
clearTimeout(timer);
timer = undefined;
}
unsubscribe();
resolve(scheme);
};
unsubscribe = this.onTerminalColorSchemeChange(settle);
timer = setTimeout(() => settle(undefined), timeoutMs);
this.terminal.write("\x1b[?996n");
this.pendingTerminalColorQueries.push(query);
this.terminal.write(TERMINAL_COLOR_QUERY);
});
}
}

View file

@ -1,6 +1,16 @@
import assert from "node:assert";
import { describe, it } from "node:test";
import { colorToRgb, indexedColor, oklchColor, parseColor, rgbColor, styleText } from "../src/index.ts";
import {
colorToHex,
colorToOkhsl,
colorToRgb,
indexedColor,
okhslColor,
oklchColor,
parseColor,
rgbColor,
styleText,
} from "../src/index.ts";
describe("colors", () => {
it("parses hex and OKLCH colors and rejects everything else", () => {
@ -16,6 +26,17 @@ describe("colors", () => {
assert.deepStrictEqual(colorToRgb(oklchColor(0, 0.3, 150)), { r: 0, g: 0, b: 0 });
});
it("parses OKHSL colors and round-trips them", () => {
// Full saturation at the red cusp is pure sRGB red.
assert.deepStrictEqual(parseColor("okhsl(29.23 100% 56.8%)"), rgbColor(255, 0, 0));
assert.deepStrictEqual(parseColor("OKHSL(250deg 60% 55%)"), okhslColor(250, 0.6, 0.55));
assert.throws(() => parseColor("okhsl(250 160% 55%)"), /s must be between 0 and 1/);
for (const hex of ["#4f8eb3", "#20242a", "#f8f9fa"]) {
const { h, s, l } = colorToOkhsl(parseColor(hex));
assert.strictEqual(colorToHex(okhslColor(h, s, l)), hex);
}
});
it("styles text and closes sequences in reverse order", () => {
assert.strictEqual(
styleText("Ready", { fg: rgbColor(18, 52, 86), bg: indexedColor(9), bold: true, italic: true }, "truecolor"),

View file

@ -59,6 +59,29 @@ class InputOverlay extends Container {
}
}
/** A settings submenu that routes keys to its nested list, like the coding agent's theme submenu. */
class SubmenuHost extends Container {
readonly list = new SelectList(
[
{ value: "first", label: "First" },
{ value: "second", label: "Second" },
],
5,
selectTheme,
);
constructor(done: (value?: string) => void) {
super();
this.list.onSelect = (item) => done(item.value);
this.list.onCancel = () => done();
this.addChild(this.list);
}
handleInput(data: string): void {
this.list.handleInput(data);
}
}
describe("mouse-aware components", () => {
it("positions a single-line input cursor on press", () => {
const input = new Input();
@ -202,6 +225,45 @@ describe("mouse-aware components", () => {
tui.stop();
});
it("keeps a settings list focused when a click in its submenu closes the submenu", async () => {
const terminal = new VirtualTerminal(30, 6);
const tui = new TuiAltScreen(terminal);
const changes: Array<{ id: string; value: string }> = [];
const list = new SettingsList(
[
{ id: "theme", label: "Theme", currentValue: "first", submenu: (_value, done) => new SubmenuHost(done) },
{ id: "other", label: "Other", currentValue: "off", values: ["off", "on"] },
],
5,
settingsTheme,
(id, value) => changes.push({ id, value }),
() => {},
);
tui.addChild(list);
tui.setFocus(list);
tui.start();
await terminal.waitForRender();
terminal.sendInput("\r");
await terminal.waitForRender();
// Press and release on the submenu's second row selects it and closes the submenu.
terminal.sendInput("\x1b[<0;3;2M");
terminal.sendInput("\x1b[<0;3;2m");
await terminal.waitForRender();
assert.deepStrictEqual(changes, [{ id: "theme", value: "second" }]);
assert.strictEqual(tui.getFocusedComponent(), list);
// Keys reach the visible list again instead of the closed submenu.
terminal.sendInput("\x1b[B");
terminal.sendInput("\r");
await terminal.waitForRender();
assert.deepStrictEqual(changes, [
{ id: "theme", value: "second" },
{ id: "other", value: "on" },
]);
tui.stop();
});
it("positions and focuses the multiline editor through alternate-screen dispatch", async () => {
const terminal = new VirtualTerminal(20, 6);
const tui = new TuiAltScreen(terminal);

View file

@ -2,12 +2,13 @@ import assert from "node:assert";
import { describe, it } from "node:test";
import {
type Component,
parseOsc11BackgroundColor,
parseTerminalColorSchemeReport,
type Terminal,
type TerminalColors,
type TUI,
TuiMainScreen,
} from "../src/index.ts";
import { parseOscColorResponse } from "../src/terminal-colors.ts";
class TestTerminal implements Terminal {
private inputHandler?: (data: string) => void;
@ -90,27 +91,6 @@ class InputRecorder implements Component {
const wait = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
describe("parseOsc11BackgroundColor", () => {
it("parses 16-bit OSC 11 rgb responses", () => {
assert.deepStrictEqual(parseOsc11BackgroundColor("\x1b]11;rgb:0000/8000/ffff\x07"), {
r: 0,
g: 128,
b: 255,
});
});
it("parses OSC 11 hex responses", () => {
assert.deepStrictEqual(parseOsc11BackgroundColor("\x1b]11;#ffffff\x1b\\"), { r: 255, g: 255, b: 255 });
assert.deepStrictEqual(parseOsc11BackgroundColor("\x1b]11;#000000\x07"), { r: 0, g: 0, b: 0 });
});
it("rejects non-strict OSC 11 responses", () => {
assert.strictEqual(parseOsc11BackgroundColor(`x\x1b]11;#ffffff\x07`), undefined);
assert.strictEqual(parseOsc11BackgroundColor("\x1b]10;#ffffff\x07"), undefined);
assert.strictEqual(parseOsc11BackgroundColor("\x1b]11;#ffffff\x07x"), undefined);
});
});
describe("parseTerminalColorSchemeReport", () => {
it("parses color scheme reports", () => {
assert.strictEqual(parseTerminalColorSchemeReport("\x1b[?997;1n"), "dark");
@ -123,151 +103,95 @@ describe("parseTerminalColorSchemeReport", () => {
});
});
describe("TUI.queryTerminalBackgroundColor", () => {
it("writes OSC 11 query and resolves with the parsed RGB reply", async () => {
const terminal = new TestTerminal();
const tui: TUI = new TuiMainScreen(terminal);
tui.start();
try {
const query = tui.queryTerminalBackgroundColor({ timeoutMs: 1000 });
assert.ok(terminal.writes.includes("\x1b]11;?\x07"));
terminal.sendInput("\x1b]11;#ffffff\x07");
assert.deepStrictEqual(await query, { r: 255, g: 255, b: 255 });
} finally {
tui.stop();
}
});
it("consumes OSC 11 replies before input listeners and focused component dispatch", async () => {
const terminal = new TestTerminal();
const tui: TUI = new TuiMainScreen(terminal);
const component = new InputRecorder();
const listenerInputs: string[] = [];
tui.addChild(component);
tui.setFocus(component);
tui.addInputListener((data) => {
listenerInputs.push(data);
return undefined;
describe("parseOscColorResponse", () => {
it("parses OSC 10, 11, and 4 replies", () => {
assert.deepStrictEqual(parseOscColorResponse("\x1b]10;rgb:ffff/ffff/ffff\x07"), {
target: "foreground",
rgb: { r: 255, g: 255, b: 255 },
});
tui.start();
try {
const query = tui.queryTerminalBackgroundColor({ timeoutMs: 1000 });
terminal.sendInput("\x1b]11;#000000\x07");
assert.deepStrictEqual(await query, { r: 0, g: 0, b: 0 });
assert.deepStrictEqual(listenerInputs, []);
assert.deepStrictEqual(component.inputs, []);
} finally {
tui.stop();
}
});
it("consumes unparseable strict OSC 11 replies and resolves undefined", async () => {
const terminal = new TestTerminal();
const tui: TUI = new TuiMainScreen(terminal);
const component = new InputRecorder();
const listenerInputs: string[] = [];
tui.addChild(component);
tui.setFocus(component);
tui.addInputListener((data) => {
listenerInputs.push(data);
return undefined;
assert.deepStrictEqual(parseOscColorResponse("\x1b]4;13;#ff0080\x1b\\"), {
target: 13,
rgb: { r: 255, g: 0, b: 128 },
});
tui.start();
try {
const query = tui.queryTerminalBackgroundColor({ timeoutMs: 1000 });
terminal.sendInput("\x1b]11;not-a-color\x07");
assert.strictEqual(await query, undefined);
assert.deepStrictEqual(listenerInputs, []);
assert.deepStrictEqual(component.inputs, []);
} finally {
tui.stop();
}
assert.deepStrictEqual(parseOscColorResponse("\x1b]4;1;bogus\x07"), { target: 1, rgb: undefined });
assert.strictEqual(parseOscColorResponse("\x1b]12;#ffffff\x07"), undefined);
});
});
it("dispatches non-matching input normally while waiting for an OSC 11 reply", async () => {
const terminal = new TestTerminal();
const tui: TUI = new TuiMainScreen(terminal);
const component = new InputRecorder();
const listenerInputs: string[] = [];
tui.addChild(component);
tui.setFocus(component);
tui.addInputListener((data) => {
listenerInputs.push(data);
return undefined;
});
tui.start();
const PALETTE_REPLIES = Array.from({ length: 16 }, (_, index) => `\x1b]4;${index};#000000\x07`);
const DA1 = "\x1b[?62;22c";
const BLACK = { r: 0, g: 0, b: 0 };
const WHITE = { r: 255, g: 255, b: 255 };
function setup(): { terminal: TestTerminal; tui: TUI; component: InputRecorder } {
const terminal = new TestTerminal();
const tui: TUI = new TuiMainScreen(terminal);
const component = new InputRecorder();
tui.addChild(component);
tui.setFocus(component);
tui.start();
return { terminal, tui, component };
}
describe("TUI.queryTerminalColors", () => {
it("queries all colors in one write and consumes the replies", async () => {
const { terminal, tui, component } = setup();
try {
let settled = false;
const query = tui.queryTerminalBackgroundColor({ timeoutMs: 1000 }).then((rgb) => {
settled = true;
return rgb;
});
const query = tui.queryTerminalColors({ timeoutMs: 1000 });
const written = terminal.writes.at(-1) ?? "";
assert.ok(written.startsWith("\x1b]10;?\x07\x1b]11;?\x07\x1b]4;0;?\x07") && written.endsWith("\x1b[c"));
terminal.sendInput("x");
await Promise.resolve();
assert.strictEqual(settled, false);
assert.deepStrictEqual(listenerInputs, ["x"]);
terminal.sendInput("\x1b]10;#ffffff\x07");
terminal.sendInput("\x1b]11;rgb:0000/0000/0000\x1b\\");
for (const reply of PALETTE_REPLIES) terminal.sendInput(reply);
// Resolves once every reply arrived, without waiting for DA1.
assert.deepStrictEqual(await query, {
foreground: WHITE,
background: BLACK,
palette: Array.from({ length: 16 }, () => BLACK),
});
terminal.sendInput(DA1);
assert.deepStrictEqual(component.inputs, ["x"]);
terminal.sendInput("\x1b]11;#ffffff\x07");
assert.deepStrictEqual(await query, { r: 255, g: 255, b: 255 });
} finally {
tui.stop();
}
});
it("queries the foreground with OSC 10 without mixing up OSC 11 replies", async () => {
const terminal = new TestTerminal();
const tui: TUI = new TuiMainScreen(terminal);
const component = new InputRecorder();
tui.addChild(component);
tui.setFocus(component);
tui.start();
it("resolves on DA1 with the replies that arrived, in query order", async () => {
const { terminal, tui } = setup();
try {
const foreground = tui.queryTerminalForegroundColor({ timeoutMs: 1000 });
const background = tui.queryTerminalBackgroundColor({ timeoutMs: 1000 });
assert.ok(terminal.writes.includes("\x1b]10;?\x07"));
const first = tui.queryTerminalColors({ timeoutMs: 1000 });
const second = tui.queryTerminalColors({ timeoutMs: 1000 });
terminal.sendInput("\x1b]11;#000000\x07");
// An incomplete palette is dropped.
for (const reply of PALETTE_REPLIES.slice(0, 8)) terminal.sendInput(reply);
terminal.sendInput(DA1);
terminal.sendInput(DA1);
terminal.sendInput("\x1b]11;rgb:0000/0000/0000\x07");
terminal.sendInput("\x1b]10;rgb:ffff/ffff/ffff\x07");
assert.deepStrictEqual(await foreground, { r: 255, g: 255, b: 255 });
assert.deepStrictEqual(await background, { r: 0, g: 0, b: 0 });
assert.deepStrictEqual(component.inputs, []);
assert.deepStrictEqual(await first, { foreground: undefined, background: BLACK, palette: undefined });
assert.deepStrictEqual(await second, { foreground: undefined, background: undefined, palette: undefined });
} finally {
tui.stop();
}
});
it("keeps consuming a late OSC 11 reply after timeout", async () => {
const terminal = new TestTerminal();
const tui: TUI = new TuiMainScreen(terminal);
const component = new InputRecorder();
const listenerInputs: string[] = [];
tui.addChild(component);
tui.setFocus(component);
tui.addInputListener((data) => {
listenerInputs.push(data);
return undefined;
});
tui.start();
it("reports late replies after a timeout and consumes them until DA1", async () => {
const { terminal, tui, component } = setup();
try {
const query = tui.queryTerminalBackgroundColor({ timeoutMs: 1 });
const late: TerminalColors[] = [];
const query = tui.queryTerminalColors({ timeoutMs: 1, onLateReply: (colors) => late.push(colors) });
await wait(5);
assert.strictEqual(await query, undefined);
assert.strictEqual((await query).background, undefined);
terminal.sendInput("\x1b]11;#ffffff\x07");
assert.deepStrictEqual(listenerInputs, []);
terminal.sendInput(DA1);
assert.deepStrictEqual(late, [{ foreground: undefined, background: WHITE, palette: undefined }]);
assert.deepStrictEqual(component.inputs, []);
// With no query pending, color replies are ordinary input again.
terminal.sendInput("\x1b]11;#ffffff\x07");
assert.deepStrictEqual(component.inputs, ["\x1b]11;#ffffff\x07"]);
} finally {
tui.stop();
}

View file

@ -187,6 +187,21 @@ describe("ProcessTerminal Kitty keyboard protocol negotiation", () => {
}
});
it("forwards device attributes replies that answer other queries", () => {
const harness = setupNegotiation();
try {
harness.send("\x1b[?7u");
harness.send("\x1b[?62;4;52c");
assert.equal(harness.getInput(), undefined);
// The TUI's color query uses DA1 as its own sentinel.
harness.send("\x1b[?62;4;52c");
assert.equal(harness.getInput(), "\x1b[?62;4;52c");
} finally {
harness.cleanup();
}
});
it("forwards normal input while waiting for Kitty response", () => {
const harness = setupNegotiation();
try {