fix(ai): omit empty text parts from multimodal user messages

Prevent OpenAI-compatible providers from rejecting image-only prompts.

Fixes #9797
This commit is contained in:
David Brailovsky 2026-09-21 21:48:11 +02:00
parent 4c8eb393c7
commit 1b6ddca87c
3 changed files with 50 additions and 15 deletions

View file

@ -2,6 +2,10 @@
## [Unreleased]
### Fixed
- Fixed image-only user messages being rejected by some OpenAI-compatible providers because they included an empty text part ([#9797](https://github.com/earendil-works/pi/issues/9797))
## [0.87.0] - 2026-09-21
### Added

View file

@ -1257,21 +1257,23 @@ export function convertMessages(
content: sanitizeSurrogates(msg.content),
});
} else {
const content: ChatCompletionContentPart[] = msg.content.map((item): ChatCompletionContentPart => {
if (item.type === "text") {
return {
type: "text",
text: sanitizeSurrogates(item.text),
} satisfies ChatCompletionContentPartText;
} else {
return {
type: "image_url",
image_url: {
url: `data:${item.mimeType};base64,${item.data}`,
},
} satisfies ChatCompletionContentPartImage;
}
});
const content: ChatCompletionContentPart[] = msg.content
.filter((item) => item.type !== "text" || item.text.length > 0)
.map((item): ChatCompletionContentPart => {
if (item.type === "text") {
return {
type: "text",
text: sanitizeSurrogates(item.text),
} satisfies ChatCompletionContentPartText;
} else {
return {
type: "image_url",
image_url: {
url: `data:${item.mimeType};base64,${item.data}`,
},
} satisfies ChatCompletionContentPartImage;
}
});
if (content.length === 0) continue;
params.push({
role: "user",

View file

@ -69,6 +69,35 @@ function buildEmptyToolResult(toolCallId: string, timestamp: number): ToolResult
}
describe("openai-completions convertMessages", () => {
// Regression test for https://github.com/earendil-works/pi/issues/9797
it("omits empty text parts from user messages with images", () => {
const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini");
const model: Model<"openai-completions"> = {
...baseModel,
api: "openai-completions",
input: ["text", "image"],
};
const context = normalizeContext({
messages: [
{
role: "user",
content: [
{ type: "text", text: "" },
{ type: "image", data: "ZmFrZQ==", mimeType: "image/png" },
],
timestamp: Date.now(),
},
],
});
expect(convertMessages(model, context, compat)).toEqual([
{
role: "user",
content: [{ type: "image_url", image_url: { url: "data:image/png;base64,ZmFrZQ==" } }],
},
]);
});
it("batches tool-result images after consecutive tool results", () => {
const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini");
const model: Model<"openai-completions"> = {