diff --git a/packages/ai/src/api/anthropic-messages.ts b/packages/ai/src/api/anthropic-messages.ts index 1e8269a62..713d273c4 100644 --- a/packages/ai/src/api/anthropic-messages.ts +++ b/packages/ai/src/api/anthropic-messages.ts @@ -706,6 +706,7 @@ export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( } } else if (event.type === "message_delta") { if (event.delta.stop_reason) { + output.rawStopReason = event.delta.stop_reason; const stopReasonResult = mapStopReason(event.delta.stop_reason, event.delta.stop_details); output.stopReason = stopReasonResult.stopReason; if (stopReasonResult.errorMessage) { @@ -1342,7 +1343,7 @@ function mapStopReason( case "stop_sequence": return { stopReason: "stop" }; // We don't supply stop sequences, so this should never happen case "sensitive": // Content flagged by safety filters (not yet in SDK types) - return { stopReason: "error" }; + return { stopReason: "error", errorMessage: "Provider stopped with: sensitive" }; default: // Handle unknown stop reasons gracefully (API may add new values) throw new Error(`Unhandled stop reason: ${reason}`); diff --git a/packages/ai/src/api/bedrock-converse-stream.ts b/packages/ai/src/api/bedrock-converse-stream.ts index efa930e42..6b767f48d 100644 --- a/packages/ai/src/api/bedrock-converse-stream.ts +++ b/packages/ai/src/api/bedrock-converse-stream.ts @@ -267,6 +267,7 @@ export const stream: StreamFunction<"bedrock-converse-stream", BedrockOptions> = } else if (item.contentBlockStop) { handleContentBlockStop(item.contentBlockStop, blocks, output, stream); } else if (item.messageStop) { + output.rawStopReason = item.messageStop.stopReason; const { stopReason, errorMessage } = mapStopReason(item.messageStop.stopReason); output.stopReason = stopReason; if (errorMessage) { @@ -969,7 +970,9 @@ function mapStopReason(reason: string | undefined): { stopReason: StopReason; er case BedrockStopReason.TOOL_USE: return { stopReason: "toolUse" }; default: - return reason ? { stopReason: "error", errorMessage: reason } : { stopReason: "error" }; + return reason + ? { stopReason: "error", errorMessage: `Provider stopped with: ${reason}` } + : { stopReason: "error" }; } } diff --git a/packages/ai/src/api/google-generative-ai.ts b/packages/ai/src/api/google-generative-ai.ts index b29244752..2e2b1784a 100644 --- a/packages/ai/src/api/google-generative-ai.ts +++ b/packages/ai/src/api/google-generative-ai.ts @@ -212,6 +212,7 @@ export const stream: StreamFunction<"google-generative-ai", GoogleOptions> = ( } if (candidate?.finishReason) { + output.rawStopReason = candidate.finishReason; output.stopReason = mapStopReason(candidate.finishReason); if (output.content.some((b) => b.type === "toolCall")) { output.stopReason = "toolUse"; @@ -266,7 +267,10 @@ export const stream: StreamFunction<"google-generative-ai", GoogleOptions> = ( throw new Error("Google stream ended without a finish reason"); } if (output.stopReason === "aborted" || output.stopReason === "error") { - throw new Error("An unknown error occurred"); + const errorMessage = output.rawStopReason + ? `Provider stopped with: ${output.rawStopReason}` + : "An unknown error occurred"; + throw new Error(errorMessage); } stream.push({ type: "done", reason: output.stopReason, message: output }); diff --git a/packages/ai/src/api/google-vertex.ts b/packages/ai/src/api/google-vertex.ts index dbef8c2f1..7baa872d1 100644 --- a/packages/ai/src/api/google-vertex.ts +++ b/packages/ai/src/api/google-vertex.ts @@ -229,6 +229,7 @@ export const stream: StreamFunction<"google-vertex", GoogleVertexOptions> = ( } if (candidate?.finishReason) { + output.rawStopReason = candidate.finishReason; output.stopReason = mapStopReason(candidate.finishReason); if (output.content.some((b) => b.type === "toolCall")) { output.stopReason = "toolUse"; @@ -283,7 +284,10 @@ export const stream: StreamFunction<"google-vertex", GoogleVertexOptions> = ( throw new Error("Google Vertex stream ended without a finish reason"); } if (output.stopReason === "aborted" || output.stopReason === "error") { - throw new Error("An unknown error occurred"); + const errorMessage = output.rawStopReason + ? `Provider stopped with: ${output.rawStopReason}` + : "An unknown error occurred"; + throw new Error(errorMessage); } stream.push({ type: "done", reason: output.stopReason, message: output }); diff --git a/packages/ai/src/api/mistral-conversations.ts b/packages/ai/src/api/mistral-conversations.ts index 160094381..9bbb96b79 100644 --- a/packages/ai/src/api/mistral-conversations.ts +++ b/packages/ai/src/api/mistral-conversations.ts @@ -89,7 +89,7 @@ export const stream: StreamFunction<"mistral-conversations", MistralOptions> = ( throw new Error("Mistral stream ended without a finish reason"); } if (output.stopReason === "aborted" || output.stopReason === "error") { - throw new Error("An unknown error occurred"); + throw new Error(output.errorMessage || "An unknown error occurred"); } stream.push({ type: "done", reason: output.stopReason, message: output }); @@ -353,7 +353,12 @@ async function consumeChatStream( if (!choice) continue; if (choice.finishReason) { - output.stopReason = mapChatStopReason(choice.finishReason); + output.rawStopReason = choice.finishReason; + const stopReasonResult = mapChatStopReason(choice.finishReason); + output.stopReason = stopReasonResult.stopReason; + if (stopReasonResult.errorMessage) { + output.errorMessage = stopReasonResult.errorMessage; + } } const delta = choice.delta; @@ -654,19 +659,19 @@ function mapToolChoice( }; } -function mapChatStopReason(reason: string | null): StopReason { - if (reason === null) return "stop"; +function mapChatStopReason(reason: string | null): { stopReason: StopReason; errorMessage?: string } { + if (reason === null) return { stopReason: "stop" }; switch (reason) { case "stop": - return "stop"; + return { stopReason: "stop" }; case "length": case "model_length": - return "length"; + return { stopReason: "length" }; case "tool_calls": - return "toolUse"; + return { stopReason: "toolUse" }; case "error": - return "error"; + return { stopReason: "error", errorMessage: "Provider stopped with: error" }; default: - return "stop"; + return { stopReason: "error", errorMessage: `Provider stopped with: ${reason}` }; } } diff --git a/packages/ai/src/api/openai-completions.ts b/packages/ai/src/api/openai-completions.ts index 410d5fa5e..436572e0f 100644 --- a/packages/ai/src/api/openai-completions.ts +++ b/packages/ai/src/api/openai-completions.ts @@ -457,6 +457,7 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio } if (choice.finish_reason) { + output.rawStopReason = choice.finish_reason; const finishReasonResult = mapStopReason(choice.finish_reason); output.stopReason = finishReasonResult.stopReason; if (finishReasonResult.errorMessage) { diff --git a/packages/ai/src/api/openai-responses-shared.ts b/packages/ai/src/api/openai-responses-shared.ts index 1b161578a..fbb11c225 100644 --- a/packages/ai/src/api/openai-responses-shared.ts +++ b/packages/ai/src/api/openai-responses-shared.ts @@ -563,7 +563,9 @@ export async function processResponsesStream( options.applyServiceTierPricing(output.usage, serviceTier); } // Map status to stop reason - output.stopReason = mapStopReason(response?.status); + const status = response?.status; + output.rawStopReason = status; + output.stopReason = mapStopReason(status); if (output.content.some((b) => b.type === "toolCall") && output.stopReason === "stop") { output.stopReason = "toolUse"; } @@ -716,6 +718,7 @@ export async function processResponsesStream( throw new Error(`Error Code ${event.code}: ${event.message}` || "Unknown error"); } else if (event.type === "response.failed") { sawTerminalResponseEvent = true; + output.rawStopReason = event.response?.status; const error = event.response?.error; const details = event.response?.incomplete_details; const msg = error diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 98cd9d22b..26470a458 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -408,6 +408,7 @@ export interface AssistantMessage { usage: Usage; stopReason: StopReason; errorMessage?: string; + rawStopReason?: string; timestamp: number; // Unix timestamp in milliseconds } diff --git a/packages/ai/test/anthropic-sse-parsing.test.ts b/packages/ai/test/anthropic-sse-parsing.test.ts index 0cdd0577c..dda6c5a07 100644 --- a/packages/ai/test/anthropic-sse-parsing.test.ts +++ b/packages/ai/test/anthropic-sse-parsing.test.ts @@ -221,9 +221,60 @@ describe("Anthropic raw SSE parsing", () => { const result = await stream.result(); expect(result.stopReason).toBe("error"); + expect(result.rawStopReason).toBe("refusal"); expect(result.errorMessage).toBe(explanation); }); + it("preserves sensitive stop reasons with a descriptive error message", async () => { + const model = getModel("anthropic", "claude-haiku-4-5"); + const context: Context = { + messages: [{ role: "user", content: "blocked request", timestamp: Date.now() }], + }; + const response = createSseResponse([ + { + event: "message_start", + data: JSON.stringify({ + type: "message_start", + message: { + id: "msg_sensitive", + usage: { + input_tokens: 12, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + }, + }), + }, + { + event: "message_delta", + data: JSON.stringify({ + type: "message_delta", + delta: { stop_reason: "sensitive" }, + usage: { + input_tokens: 12, + output_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + }), + }, + { + event: "message_stop", + data: JSON.stringify({ type: "message_stop" }), + }, + ]); + + const stream = streamAnthropic(model, context, { + client: createFakeAnthropicClient(response), + }); + const result = await stream.result(); + + expect(result.stopReason).toBe("error"); + expect(result.rawStopReason).toBe("sensitive"); + expect(result.errorMessage).toBe("Provider stopped with: sensitive"); + }); + it("treats message_delta without usage as a no-op for usage accumulation", async () => { const model = getModel("anthropic", "claude-haiku-4-5"); const context: Context = { diff --git a/packages/ai/test/bedrock-raw-stop-reason.test.ts b/packages/ai/test/bedrock-raw-stop-reason.test.ts new file mode 100644 index 000000000..128db0498 --- /dev/null +++ b/packages/ai/test/bedrock-raw-stop-reason.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it, vi } from "vitest"; + +const bedrockMock = vi.hoisted(() => ({ + stopReason: "end_turn" as string, +})); + +vi.mock("@aws-sdk/client-bedrock-runtime", () => { + class BedrockRuntimeServiceException extends Error {} + + class BedrockRuntimeClient { + middlewareStack = { + add: () => undefined, + }; + + async send(): Promise { + return { + $metadata: { httpStatusCode: 200, requestId: "request-id" }, + stream: (async function* () { + yield { messageStart: { role: "assistant" } }; + yield { messageStop: { stopReason: bedrockMock.stopReason } }; + })(), + }; + } + } + + class ConverseStreamCommand { + readonly input: unknown; + + constructor(input: unknown) { + this.input = input; + } + } + + return { + BedrockRuntimeClient, + BedrockRuntimeServiceException, + ConverseStreamCommand, + StopReason: { + END_TURN: "end_turn", + STOP_SEQUENCE: "stop_sequence", + MAX_TOKENS: "max_tokens", + MODEL_CONTEXT_WINDOW_EXCEEDED: "model_context_window_exceeded", + TOOL_USE: "tool_use", + }, + CachePointType: { DEFAULT: "default" }, + CacheTTL: { ONE_HOUR: "ONE_HOUR" }, + ConversationRole: { ASSISTANT: "assistant", USER: "user" }, + ImageFormat: { JPEG: "jpeg", PNG: "png", GIF: "gif", WEBP: "webp" }, + ToolResultStatus: { ERROR: "error", SUCCESS: "success" }, + }; +}); + +import { stream as streamBedrock } from "../src/api/bedrock-converse-stream.ts"; +import { getModel } from "../src/compat.ts"; +import type { Context } from "../src/types.ts"; + +const model = getModel("amazon-bedrock", "us.anthropic.claude-opus-4-8"); +const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: Date.now() }], +}; + +describe("Bedrock raw stop reasons", () => { + it("preserves raw Bedrock stop reasons for successful stops", async () => { + bedrockMock.stopReason = "end_turn"; + + const message = await streamBedrock(model, context, { cacheRetention: "none" }).result(); + + expect(message.stopReason).toBe("stop"); + expect(message.rawStopReason).toBe("end_turn"); + expect(message.errorMessage).toBeUndefined(); + }); + + it("preserves raw Bedrock stop reasons for provider error stops", async () => { + bedrockMock.stopReason = "guardrail_intervened"; + + const message = await streamBedrock(model, context, { cacheRetention: "none" }).result(); + + expect(message.stopReason).toBe("error"); + expect(message.rawStopReason).toBe("guardrail_intervened"); + expect(message.errorMessage).toBe("Provider stopped with: guardrail_intervened"); + }); +}); diff --git a/packages/ai/test/google-raw-stop-reason.test.ts b/packages/ai/test/google-raw-stop-reason.test.ts new file mode 100644 index 000000000..b485088ea --- /dev/null +++ b/packages/ai/test/google-raw-stop-reason.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it, vi } from "vitest"; + +const googleGenAiMock = vi.hoisted(() => ({ + finishReason: "MALFORMED_FUNCTION_CALL", +})); + +vi.mock("@google/genai", () => { + class GoogleGenAI { + models = { + generateContentStream: async function* () { + yield { + responseId: "google-response-id", + candidates: [ + { + finishReason: googleGenAiMock.finishReason, + }, + ], + usageMetadata: { + promptTokenCount: 1, + candidatesTokenCount: 0, + totalTokenCount: 1, + }, + }; + }, + }; + } + + return { + FinishReason: { + STOP: "STOP", + MAX_TOKENS: "MAX_TOKENS", + BLOCKLIST: "BLOCKLIST", + PROHIBITED_CONTENT: "PROHIBITED_CONTENT", + SPII: "SPII", + SAFETY: "SAFETY", + IMAGE_SAFETY: "IMAGE_SAFETY", + IMAGE_PROHIBITED_CONTENT: "IMAGE_PROHIBITED_CONTENT", + IMAGE_RECITATION: "IMAGE_RECITATION", + IMAGE_OTHER: "IMAGE_OTHER", + RECITATION: "RECITATION", + FINISH_REASON_UNSPECIFIED: "FINISH_REASON_UNSPECIFIED", + OTHER: "OTHER", + LANGUAGE: "LANGUAGE", + MALFORMED_FUNCTION_CALL: "MALFORMED_FUNCTION_CALL", + UNEXPECTED_TOOL_CALL: "UNEXPECTED_TOOL_CALL", + NO_IMAGE: "NO_IMAGE", + }, + FunctionCallingConfigMode: { + AUTO: "AUTO", + NONE: "NONE", + ANY: "ANY", + VALIDATED: "VALIDATED", + }, + GoogleGenAI, + ResourceScope: { + COLLECTION: "COLLECTION", + }, + ThinkingLevel: { + THINKING_LEVEL_UNSPECIFIED: "THINKING_LEVEL_UNSPECIFIED", + MINIMAL: "MINIMAL", + LOW: "LOW", + MEDIUM: "MEDIUM", + HIGH: "HIGH", + }, + }; +}); + +import { stream as streamGoogleGenerativeAi } from "../src/api/google-generative-ai.ts"; +import { stream as streamGoogleVertex } from "../src/api/google-vertex.ts"; +import { getModel } from "../src/compat.ts"; +import type { Context } from "../src/types.ts"; + +const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: Date.now() }], +}; + +describe("Google raw stop reasons", () => { + it("preserves raw Gemini finish reasons for Google Generative AI errors", async () => { + googleGenAiMock.finishReason = "MALFORMED_FUNCTION_CALL"; + + const stream = streamGoogleGenerativeAi(getModel("google", "gemini-2.5-flash"), context, { + apiKey: "test-api-key", + }); + + const message = await stream.result(); + + expect(message.stopReason).toBe("error"); + expect(message.rawStopReason).toBe("MALFORMED_FUNCTION_CALL"); + expect(message.errorMessage).toBe("Provider stopped with: MALFORMED_FUNCTION_CALL"); + }); + + it("preserves raw Gemini finish reasons for Google Vertex errors", async () => { + googleGenAiMock.finishReason = "SAFETY"; + + const stream = streamGoogleVertex(getModel("google-vertex", "gemini-3-flash-preview"), context, { + project: "test-project", + location: "us-central1", + }); + + const message = await stream.result(); + + expect(message.stopReason).toBe("error"); + expect(message.rawStopReason).toBe("SAFETY"); + expect(message.errorMessage).toBe("Provider stopped with: SAFETY"); + }); +}); diff --git a/packages/ai/test/mistral-raw-stop-reason.test.ts b/packages/ai/test/mistral-raw-stop-reason.test.ts new file mode 100644 index 000000000..782e057a2 --- /dev/null +++ b/packages/ai/test/mistral-raw-stop-reason.test.ts @@ -0,0 +1,77 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mistralMock = vi.hoisted(() => ({ + finishReason: "stop" as string, +})); + +vi.mock("@mistralai/mistralai", () => { + class HTTPClient {} + + class Mistral { + chat = { + stream: async function* () { + yield { + data: { + id: "mistral-response-id", + choices: [ + { + finishReason: mistralMock.finishReason, + delta: {}, + }, + ], + usage: { + promptTokens: 1, + completionTokens: 0, + totalTokens: 1, + }, + }, + }; + }, + }; + } + + return { HTTPClient, Mistral }; +}); + +import { stream as streamMistral } from "../src/api/mistral-conversations.ts"; +import { getModel } from "../src/compat.ts"; +import type { Context } from "../src/types.ts"; + +const model = getModel("mistral", "devstral-medium-latest"); +const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: Date.now() }], +}; + +describe("Mistral raw stop reasons", () => { + beforeEach(() => { + mistralMock.finishReason = "stop"; + }); + + it("preserves raw Mistral finish reasons for successful stops", async () => { + const message = await streamMistral(model, context, { apiKey: "test" }).result(); + + expect(message.stopReason).toBe("stop"); + expect(message.rawStopReason).toBe("stop"); + expect(message.errorMessage).toBeUndefined(); + }); + + it("preserves raw Mistral finish reasons for provider error stops", async () => { + mistralMock.finishReason = "error"; + + const message = await streamMistral(model, context, { apiKey: "test" }).result(); + + expect(message.stopReason).toBe("error"); + expect(message.rawStopReason).toBe("error"); + expect(message.errorMessage).toBe("Provider stopped with: error"); + }); + + it("treats unknown Mistral finish reasons as provider error stops", async () => { + mistralMock.finishReason = "unmapped_error"; + + const message = await streamMistral(model, context, { apiKey: "test" }).result(); + + expect(message.stopReason).toBe("error"); + expect(message.rawStopReason).toBe("unmapped_error"); + expect(message.errorMessage).toBe("Provider stopped with: unmapped_error"); + }); +}); diff --git a/packages/ai/test/openai-completions-raw-stop-reason.test.ts b/packages/ai/test/openai-completions-raw-stop-reason.test.ts new file mode 100644 index 000000000..5a51909f9 --- /dev/null +++ b/packages/ai/test/openai-completions-raw-stop-reason.test.ts @@ -0,0 +1,79 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { stream as streamOpenAICompletions } from "../src/api/openai-completions.ts"; +import type { Context, Model } from "../src/types.ts"; + +const mockState = vi.hoisted(() => ({ + chunks: [] as unknown[], +})); + +vi.mock("openai", () => { + class FakeOpenAI { + chat = { + completions: { + create: () => { + const chunks = mockState.chunks; + const stream = { + async *[Symbol.asyncIterator]() { + for (const chunk of chunks) yield chunk; + }, + }; + const promise = Promise.resolve(stream) as Promise & { + withResponse: () => Promise<{ + data: typeof stream; + response: { status: number; headers: Headers }; + }>; + }; + promise.withResponse = async () => ({ + data: stream, + response: { status: 200, headers: new Headers() }, + }); + return promise; + }, + }, + }; + } + return { default: FakeOpenAI }; +}); + +const model: Model<"openai-completions"> = { + id: "test-model", + name: "Test Model", + api: "openai-completions", + provider: "openai", + baseUrl: "https://api.openai.com/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 4096, +}; + +const context: Context = { + messages: [{ role: "user", content: "hello", timestamp: Date.now() }], +}; + +describe("OpenAI completions raw stop reasons", () => { + beforeEach(() => { + mockState.chunks = []; + }); + + it("preserves raw finish reasons for successful stops", async () => { + mockState.chunks = [{ id: "chatcmpl-1", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }]; + + const message = await streamOpenAICompletions(model, context, { apiKey: "test" }).result(); + + expect(message.stopReason).toBe("stop"); + expect(message.rawStopReason).toBe("stop"); + expect(message.errorMessage).toBeUndefined(); + }); + + it("preserves raw finish reasons for provider error stops", async () => { + mockState.chunks = [{ id: "chatcmpl-2", choices: [{ index: 0, delta: {}, finish_reason: "content_filter" }] }]; + + const message = await streamOpenAICompletions(model, context, { apiKey: "test" }).result(); + + expect(message.stopReason).toBe("error"); + expect(message.rawStopReason).toBe("content_filter"); + expect(message.errorMessage).toBe("Provider finish_reason: content_filter"); + }); +}); diff --git a/packages/ai/test/openai-responses-terminal-event.test.ts b/packages/ai/test/openai-responses-terminal-event.test.ts index fcc3cc522..2e02151e0 100644 --- a/packages/ai/test/openai-responses-terminal-event.test.ts +++ b/packages/ai/test/openai-responses-terminal-event.test.ts @@ -285,6 +285,7 @@ describe("OpenAI Responses terminal event handling", () => { expect(output.responseId).toBe("resp_completed"); expect(output.stopReason).toBe("stop"); + expect(output.rawStopReason).toBe("completed"); expect(output.usage).toMatchObject({ input: 15, output: 7, @@ -303,6 +304,7 @@ describe("OpenAI Responses terminal event handling", () => { expect(output.responseId).toBe("resp_incomplete"); expect(output.stopReason).toBe("length"); + expect(output.rawStopReason).toBe("incomplete"); expect(output.usage).toMatchObject({ input: 25, output: 12, @@ -320,5 +322,6 @@ describe("OpenAI Responses terminal event handling", () => { await expect(processResponsesStream(createFailedEvents(), output, stream, model)).rejects.toThrow( "server_error: boom", ); + expect(output.rawStopReason).toBe("failed"); }); });