pi-mono/packages/coding-agent/test/assistant-message.test.ts
David Brailovsky 32850ef7c5
fix(coding-agent): resume after context-limited length stops (#7540)
* fix(coding-agent): resume after context-limited length stops

Treat length stops as context overflow when prompt usage is within 1% of
Pi's configured context window. Include cache-write tokens in that usage
and allow non-zero output because OpenAI may emit reasoning tokens before
returning an incomplete response.

After compaction, remove any retryable error or truncated-length assistant
message restored from persisted session history so the interrupted agent
turn can continue.

This is specifically for handling this case for openai models:
https://developers.openai.com/api/docs/guides/reasoning

> If the generated tokens reach the context window limit or the max_output_tokens value you’ve set, you’ll receive a response with a status of incomplete and incomplete_details with reason set to max_output_tokens. This might occur before any visible output tokens are produced, meaning you could incur costs for input and reasoning tokens without receiving a visible response.

This detection currently assumes the reported prompt usage is within 1% of
the context window configured in Pi. Its reliability therefore depends on
Pi's model contextWindow metadata matching the provider's effective limit.

Maybe we should consider treating OpenAI incomplete responses as requiring compaction
regardless of the local context-window estimate?

fixes #7020

* fix(coding-agent): recover truncated responses after compaction

Classify length stops by comparing provider-reported output usage with the model's intended output limit. Compact and retry once when generation ends below that limit, avoiding unreliable context-window estimates while preserving genuine max-output stops.

Normalize OpenAI Responses incomplete reasons so only max_output_tokens is treated as a length stop. Preserve qualified provider reasons for diagnostics, surface other reasons as errors, and use neutral truncation wording in the TUI.
2026-08-03 18:07:53 +02:00

241 lines
8 KiB
TypeScript

import type { AssistantMessage } from "@earendil-works/pi-ai";
import { describe, expect, test } from "vitest";
import { AssistantMessageComponent } from "../src/modes/interactive/components/assistant-message.ts";
import { UserMessageComponent } from "../src/modes/interactive/components/user-message.ts";
import { initTheme } from "../src/modes/interactive/theme/theme.ts";
import { stripAnsi } from "../src/utils/ansi.ts";
const OSC133_ZONE_START = "\x1b]133;A\x07";
const OSC133_ZONE_END = "\x1b]133;B\x07";
const OSC133_ZONE_FINAL = "\x1b]133;C\x07";
function createAssistantMessage(
content: AssistantMessage["content"],
overrides: Partial<Pick<AssistantMessage, "stopReason">> = {},
): AssistantMessage {
return {
role: "assistant",
content,
api: "openai-responses",
provider: "openai",
model: "gpt-4o-mini",
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: overrides.stopReason ?? "stop",
timestamp: Date.now(),
};
}
describe("AssistantMessageComponent", () => {
test("adds OSC 133 zone markers to assistant messages without tool calls", () => {
initTheme("dark");
const component = new AssistantMessageComponent(createAssistantMessage([{ type: "text", text: "hello" }]));
const lines = component.render(40);
expect(lines).not.toHaveLength(0);
expect(lines[0]).toContain(OSC133_ZONE_START);
expect(lines[lines.length - 1].startsWith(OSC133_ZONE_END + OSC133_ZONE_FINAL)).toBe(true);
});
test("does not add OSC 133 zone markers when assistant message contains tool calls", () => {
initTheme("dark");
const component = new AssistantMessageComponent(
createAssistantMessage([
{ type: "text", text: "calling tool" },
{ type: "toolCall", id: "tool-1", name: "read", arguments: { path: "file.txt" } },
]),
);
const rendered = component.render(60).join("\n");
expect(rendered.includes(OSC133_ZONE_START)).toBe(false);
expect(rendered.includes(OSC133_ZONE_END)).toBe(false);
expect(rendered.includes(OSC133_ZONE_FINAL)).toBe(false);
});
test("renders length stops with neutral truncation wording", () => {
initTheme("dark");
const component = new AssistantMessageComponent(
createAssistantMessage([{ type: "thinking", thinking: "private reasoning" }], { stopReason: "length" }),
true,
);
const rendered = component.render(80).join("\n");
expect(rendered).toContain("Thinking...");
expect(rendered).toContain("Response was truncated before completion.");
});
test("coalesces adjacent thinking blocks into one hidden thinking label", () => {
initTheme("dark");
const component = new AssistantMessageComponent(
createAssistantMessage([
{ type: "thinking", thinking: "first thought" },
{ type: "thinking", thinking: "" },
{ type: "thinking", thinking: "second thought" },
{ type: "text", text: "answer" },
]),
true,
);
const rendered = stripAnsi(component.render(80).join("\n"));
expect(rendered.match(/Thinking\.\.\./g)).toHaveLength(1);
expect(rendered).toContain("answer");
});
test("uses configured output padding for text and thinking", () => {
initTheme("dark");
const component = new AssistantMessageComponent(
createAssistantMessage([
{ type: "text", text: "hello" },
{ type: "thinking", thinking: "reasoning" },
]),
false,
undefined,
"Thinking...",
1,
);
const lines = component.render(80).map((line) => stripAnsi(line));
expect(lines.some((line) => line.includes(" hello"))).toBe(true);
expect(lines.some((line) => line.includes(" reasoning"))).toBe(true);
component.setOutputPad(0);
const updatedLines = component.render(80).map((line) => stripAnsi(line));
expect(updatedLines.some((line) => line.startsWith("hello"))).toBe(true);
expect(updatedLines.some((line) => line.startsWith("reasoning"))).toBe(true);
});
test("chains Markdown transformers in registration order", () => {
initTheme("dark");
const calls: string[] = [];
const message = createAssistantMessage([{ type: "text", text: "The result is $x^2$." }]);
const component = new AssistantMessageComponent(message, false, undefined, "Thinking...", 1, [
(markdown, context) => {
calls.push("formula");
expect(context).toEqual({ messageType: "assistant", isStreaming: false, availableWidth: 78 });
return markdown.replace("$x^2$", "x²");
},
(markdown) => {
calls.push("suffix");
return `${markdown} Done.`;
},
]);
expect(stripAnsi(component.render(80).join("\n"))).toContain("The result is x². Done.");
expect(calls).toEqual(["formula", "suffix"]);
});
test("identifies partial assistant Markdown as streaming", () => {
initTheme("dark");
const streamingStates: boolean[] = [];
const message = createAssistantMessage([{ type: "text", text: "partial" }]);
const component = new AssistantMessageComponent(undefined, false, undefined, "Thinking...", 1, [
(markdown, context) => {
streamingStates.push(context.isStreaming);
return context.isStreaming ? markdown : `${markdown} transformed`;
},
]);
component.updateContent(message, true);
expect(stripAnsi(component.render(80).join("\n"))).not.toContain("transformed");
component.updateContent(message, false);
expect(stripAnsi(component.render(80).join("\n"))).toContain("partial transformed");
expect(streamingStates).toEqual([true, false]);
});
test("reapplies Markdown transformers when available width changes", () => {
initTheme("dark");
const availableWidths: number[] = [];
const component = new AssistantMessageComponent(
createAssistantMessage([{ type: "text", text: "answer" }]),
false,
undefined,
"Thinking...",
1,
[
(markdown, context) => {
availableWidths.push(context.availableWidth);
return `${markdown} (${context.availableWidth})`;
},
],
);
expect(stripAnsi(component.render(80).join("\n"))).toContain("answer (78)");
component.render(80);
expect(stripAnsi(component.render(60).join("\n"))).toContain("answer (58)");
expect(availableWidths).toEqual([78, 58]);
});
test("continues the Markdown transformer chain when a transformer throws", () => {
initTheme("dark");
const calls: string[] = [];
const component = new AssistantMessageComponent(
createAssistantMessage([{ type: "text", text: "still visible" }]),
false,
undefined,
"Thinking...",
1,
[
(markdown) => {
calls.push("first");
return markdown.replace("still", "remains");
},
() => {
calls.push("throw");
throw new Error("broken transformer");
},
(markdown) => {
calls.push("last");
return `${markdown} after error`;
},
],
);
expect(stripAnsi(component.render(80).join("\n"))).toContain("remains visible after error");
expect(calls).toEqual(["first", "throw", "last"]);
});
test("transforms text and thinking Markdown without mutating the original message", () => {
initTheme("dark");
const message = createAssistantMessage([
{ type: "text", text: "answer" },
{ type: "thinking", thinking: "reasoning" },
]);
const component = new AssistantMessageComponent(message, false, undefined, "Thinking...", 1, [
(markdown, { messageType }) => {
return `${messageType}:${markdown}`;
},
]);
const rendered = stripAnsi(component.render(80).join("\n"));
expect(rendered).toContain("assistant:answer");
expect(rendered).toContain("assistant-thinking:reasoning");
expect(message.content).toEqual([
{ type: "text", text: "answer" },
{ type: "thinking", thinking: "reasoning" },
]);
});
test("uses configured output padding for user messages", () => {
initTheme("dark");
const paddedComponent = new UserMessageComponent("hello", undefined, 1);
const paddedLines = paddedComponent.render(40).map((line) => stripAnsi(line));
expect(paddedLines.some((line) => line.startsWith(" hello"))).toBe(true);
const unpaddedComponent = new UserMessageComponent("hello", undefined, 0);
const unpaddedLines = unpaddedComponent.render(40).map((line) => stripAnsi(line));
expect(unpaddedLines.some((line) => line.startsWith("hello"))).toBe(true);
});
});