From 024abf6ddafbd233b6caee2f70067239c1a329b3 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sat, 24 Oct 2026 17:44:58 +0200 Subject: [PATCH] fix(ai): route Bedrock models Converse does not serve through Bedrock Mantle Bedrock serves OpenAI GPT, Grok, and Gemma 4 models only on Mantle, its OpenAI-compatible endpoint; Converse rejects them as invalid model identifiers. models.dev v2 lists each model's APIs, so the catalog now maps models without a Converse API to the Responses API at the Mantle URL, and amazon-bedrock dispatches them to a thin wrapper around the OpenAI Responses implementation that fills in the region and authenticates with a Bedrock API key. IAM and profile credentials are not supported for Mantle models yet. --- packages/ai/README.md | 2 +- .../ai/catalog/providers/amazon-bedrock.ts | 35 +++++++++-- packages/ai/src/api/bedrock-mantle.lazy.ts | 4 ++ packages/ai/src/api/bedrock-mantle.ts | 61 +++++++++++++++++++ packages/ai/src/providers/amazon-bedrock.ts | 11 +++- packages/ai/test/bedrock-mantle.test.ts | 59 ++++++++++++++++++ packages/coding-agent/docs/providers.md | 2 + scripts/model-catalog-minimum-version.mjs | 3 +- 8 files changed, 166 insertions(+), 11 deletions(-) create mode 100644 packages/ai/src/api/bedrock-mantle.lazy.ts create mode 100644 packages/ai/src/api/bedrock-mantle.ts create mode 100644 packages/ai/test/bedrock-mantle.test.ts diff --git a/packages/ai/README.md b/packages/ai/README.md index 3718ef092..b95af9368 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -556,7 +556,7 @@ The Z.AI pay-as-you-go APIs (`zai-api`, `zai-api-cn`) and Alibaba Cloud Model St subscriptions, while the existing provider retains its broader catalog for backward compatibility. Stored credentials remain provider-scoped, so save the key under the provider ID you register. -Amazon Bedrock resolves ambient AWS credentials (`AWS_PROFILE`, access key pairs, `AWS_BEARER_TOKEN_BEDROCK`, ECS task roles, web identity tokens); its provider-owned login flow supports bearer tokens, AWS profiles, and the existing credential chain. Vertex AI resolves either an explicit key or gcloud Application Default Credentials plus project/location, with a provider-owned login flow for API keys, ADC, and service-account files. +Amazon Bedrock resolves ambient AWS credentials (`AWS_PROFILE`, access key pairs, `AWS_BEARER_TOKEN_BEDROCK`, ECS task roles, web identity tokens); its provider-owned login flow supports bearer tokens, AWS profiles, and the existing credential chain. Models Bedrock only serves on Mantle, its OpenAI-compatible endpoint (for example OpenAI's GPT models), use the Responses API at `bedrock-mantle.{region}.api.aws` and currently need a Bedrock API key. Vertex AI resolves either an explicit key or gcloud Application Default Credentials plus project/location, with a provider-owned login flow for API keys, ADC, and service-account files. ## Tools diff --git a/packages/ai/catalog/providers/amazon-bedrock.ts b/packages/ai/catalog/providers/amazon-bedrock.ts index 28bc0c71c..7166ebb54 100644 --- a/packages/ai/catalog/providers/amazon-bedrock.ts +++ b/packages/ai/catalog/providers/amazon-bedrock.ts @@ -1,4 +1,6 @@ -import { defineProvider, modelsDevModels, withInputLimits } from "../define.ts"; +import type { Model } from "../../src/types.ts"; +import { type ChatModel, defineProvider, modelsDevModels, withInputLimits } from "../define.ts"; +import type { ModelsDevModel } from "../sources/models-dev.ts"; // Opus 5 is only callable through cross-region inference profiles. const INFERENCE_PROFILE_ONLY_MODEL_IDS = new Set(["anthropic.claude-opus-5"]); @@ -9,6 +11,24 @@ function bedrockBaseUrl(modelId: string): string { : "https://bedrock-runtime.us-east-1.amazonaws.com"; } +/** + * Models Bedrock serves only on Mantle, its OpenAI-compatible endpoint, are + * rejected by Converse as invalid model identifiers. They use the Responses API + * at the Mantle URL models.dev lists, with `{region}` filled in per request. + */ +function mantleModel(model: ChatModel, source: ModelsDevModel): Model<"openai-responses"> | undefined { + const responses = source.apis.find((api) => api.protocol === "responses"); + if (!responses) return undefined; + return { + ...model, + api: "openai-responses", + // models.dev writes the region as a `${AWS_REGION}` template. + baseUrl: responses.baseUrl.replace(/\$\{AWS_REGION\}/, "{region}"), + // Mantle does not document OpenAI's extended prompt cache retention. + compat: { supportsLongCacheRetention: false }, + }; +} + export default defineProvider({ id: "amazon-bedrock", models: (ctx) => @@ -22,11 +42,14 @@ export default defineProvider({ // Mistral 7B Instruct does not support system messages. !source.id.startsWith("mistral.mistral-7b-instruct-v0"), // Includes models.dev pricing tiers, e.g. the long-context tier for OpenAI models (#10326). - map: (model, source) => ({ - ...model, - baseUrl: bedrockBaseUrl(model.id), - ...(source.structuredOutput ? { compat: { supportsStrictMode: true } } : {}), - }), + map: (model, source) => { + if (!source.apis.some((api) => api.protocol === "converse")) return mantleModel(model, source); + return { + ...model, + baseUrl: bedrockBaseUrl(model.id), + ...(source.structuredOutput ? { compat: { supportsStrictMode: true } } : {}), + }; + }, }), finalize: (model) => withInputLimits(model, { images: { maxPerMessage: 20 } }), }); diff --git a/packages/ai/src/api/bedrock-mantle.lazy.ts b/packages/ai/src/api/bedrock-mantle.lazy.ts new file mode 100644 index 000000000..7c8f3c4f4 --- /dev/null +++ b/packages/ai/src/api/bedrock-mantle.lazy.ts @@ -0,0 +1,4 @@ +import type { ProviderStreams } from "../types.ts"; +import { lazyApi } from "./lazy.ts"; + +export const bedrockMantleApi = (): ProviderStreams => lazyApi(() => import("./bedrock-mantle.ts")); diff --git a/packages/ai/src/api/bedrock-mantle.ts b/packages/ai/src/api/bedrock-mantle.ts new file mode 100644 index 000000000..3b0551d8b --- /dev/null +++ b/packages/ai/src/api/bedrock-mantle.ts @@ -0,0 +1,61 @@ +// Bedrock Mantle: Amazon Bedrock's OpenAI-compatible endpoint, which serves +// models Converse does not (GPT, Grok, Gemma). This module reuses the OpenAI +// Responses implementation with the region filled into the catalog URL and a +// Bedrock API key as the bearer token. +// https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html + +import type { Model, ProviderEnv, SimpleStreamOptions, StreamFunction, TranscriptContext } from "../types.ts"; +import type { AssistantMessageEventStream } from "../utils/event-stream.ts"; +import { getProviderEnvValue } from "../utils/provider-env.ts"; +import { lazyStream } from "./lazy.ts"; +import { + type OpenAIResponsesOptions, + stream as responsesStream, + streamSimple as responsesStreamSimple, +} from "./openai-responses.ts"; + +/** The region Mantle requests go to, resolved like Bedrock Converse requests. */ +export function resolveMantleRegion(env: ProviderEnv | undefined): string { + return getProviderEnvValue("AWS_REGION", env) || getProviderEnvValue("AWS_DEFAULT_REGION", env) || "us-east-1"; +} + +/** The Bedrock API key for Mantle: a stored credential or `AWS_BEARER_TOKEN_BEDROCK`. */ +export function resolveMantleBearerToken(apiKey: string | undefined, env: ProviderEnv | undefined): string { + const token = apiKey || getProviderEnvValue("AWS_BEARER_TOKEN_BEDROCK", env); + if (!token) { + throw new Error( + "Bedrock Mantle models need an Amazon Bedrock API key. Set AWS_BEARER_TOKEN_BEDROCK or log in with a bearer token; AWS profiles and IAM credentials are not supported for these models yet.", + ); + } + return token; +} + +function prepare( + model: Model<"openai-responses">, + options: TOptions | undefined, +): { model: Model<"openai-responses">; options: TOptions } { + return { + model: { ...model, baseUrl: model.baseUrl.replace("{region}", resolveMantleRegion(options?.env)) }, + options: { ...options, apiKey: resolveMantleBearerToken(options?.apiKey, options?.env) } as TOptions, + }; +} + +export const stream: StreamFunction<"openai-responses", OpenAIResponsesOptions> = ( + model: Model<"openai-responses">, + context: TranscriptContext, + options?: OpenAIResponsesOptions, +): AssistantMessageEventStream => + lazyStream(model, async () => { + const prepared = prepare(model, options); + return responsesStream(prepared.model, context, prepared.options); + }); + +export const streamSimple: StreamFunction<"openai-responses", SimpleStreamOptions> = ( + model: Model<"openai-responses">, + context: TranscriptContext, + options?: SimpleStreamOptions, +): AssistantMessageEventStream => + lazyStream(model, async () => { + const prepared = prepare(model, options); + return responsesStreamSimple(prepared.model, context, prepared.options); + }); diff --git a/packages/ai/src/providers/amazon-bedrock.ts b/packages/ai/src/providers/amazon-bedrock.ts index d629f1e1c..128cdfbd9 100644 --- a/packages/ai/src/providers/amazon-bedrock.ts +++ b/packages/ai/src/providers/amazon-bedrock.ts @@ -1,4 +1,5 @@ import { bedrockConverseStreamApi } from "../api/bedrock-converse-stream.lazy.ts"; +import { bedrockMantleApi } from "../api/bedrock-mantle.lazy.ts"; import type { ApiKeyAuth } from "../auth/types.ts"; import { createProvider, type Provider } from "../models.ts"; import { AMAZON_BEDROCK_MODELS } from "./amazon-bedrock.models.ts"; @@ -79,12 +80,16 @@ const bedrockAuth: ApiKeyAuth = { }, }; -export function amazonBedrockProvider(): Provider<"bedrock-converse-stream"> { - return createProvider({ +/** Converse for most models; Mantle's OpenAI Responses API for models Converse does not serve. */ +export function amazonBedrockProvider(): Provider<"bedrock-converse-stream" | "openai-responses"> { + return createProvider<"bedrock-converse-stream" | "openai-responses">({ id: "amazon-bedrock", name: "Amazon Bedrock", auth: { apiKey: bedrockAuth }, models: Object.values(AMAZON_BEDROCK_MODELS), - api: bedrockConverseStreamApi(), + api: { + "bedrock-converse-stream": bedrockConverseStreamApi(), + "openai-responses": bedrockMantleApi(), + }, }); } diff --git a/packages/ai/test/bedrock-mantle.test.ts b/packages/ai/test/bedrock-mantle.test.ts new file mode 100644 index 000000000..1004633a2 --- /dev/null +++ b/packages/ai/test/bedrock-mantle.test.ts @@ -0,0 +1,59 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { streamSimple } from "../src/api/bedrock-mantle.ts"; +import { hasApi } from "../src/models.ts"; +import { getBuiltinModels } from "../src/providers/all.ts"; +import type { Model } from "../src/types.ts"; +import { normalizeContext } from "../src/utils/transcript.ts"; + +function mantleModels(): Model<"openai-responses">[] { + return getBuiltinModels("amazon-bedrock").filter((model) => hasApi(model, "openai-responses")); +} + +const context = normalizeContext({ messages: [{ role: "user", content: "hi", timestamp: 0 }] }); + +describe("Bedrock Mantle", () => { + let savedToken: string | undefined; + beforeEach(() => { + savedToken = process.env.AWS_BEARER_TOKEN_BEDROCK; + delete process.env.AWS_BEARER_TOKEN_BEDROCK; + }); + afterEach(() => { + if (savedToken === undefined) delete process.env.AWS_BEARER_TOKEN_BEDROCK; + else process.env.AWS_BEARER_TOKEN_BEDROCK = savedToken; + }); + + it("routes models Converse does not serve to the Mantle Responses endpoint", () => { + const models = mantleModels(); + expect(models.length).toBeGreaterThan(0); + for (const model of models) { + expect(model.baseUrl).toMatch(/^https:\/\/bedrock-mantle\.\{region\}\.api\.aws\//); + } + const converse = getBuiltinModels("amazon-bedrock").filter((model) => hasApi(model, "bedrock-converse-stream")); + expect(converse.some((model) => model.id.includes("anthropic.claude"))).toBe(true); + }); + + it("sends Responses requests to the regional Mantle host with the Bedrock API key", async () => { + const model = mantleModels()[0]; + const requests: { url: string; authorization: string | null }[] = []; + const result = await streamSimple(model, context, { + apiKey: "bedrock-api-key", + env: { AWS_REGION: "us-west-2" }, + fetch: async (input, init) => { + requests.push({ url: String(input), authorization: new Headers(init?.headers).get("authorization") }); + return new Response(JSON.stringify({ error: { message: "captured" } }), { status: 400 }); + }, + }).result(); + + expect(result.stopReason).toBe("error"); + expect(requests[0]).toEqual({ + url: `${model.baseUrl.replace("{region}", "us-west-2")}/responses`, + authorization: "Bearer bedrock-api-key", + }); + }); + + it("explains that Mantle models need a Bedrock API key", async () => { + const result = await streamSimple(mantleModels()[0], context, { env: {} }).result(); + expect(result.stopReason).toBe("error"); + expect(result.errorMessage).toContain("AWS_BEARER_TOKEN_BEDROCK"); + }); +}); diff --git a/packages/coding-agent/docs/providers.md b/packages/coding-agent/docs/providers.md index 7b56dc46e..3261a1f74 100644 --- a/packages/coding-agent/docs/providers.md +++ b/packages/coding-agent/docs/providers.md @@ -213,6 +213,8 @@ export AWS_REGION=us-west-2 Pi also supports ECS task credentials and IRSA through the standard `AWS_CONTAINER_CREDENTIALS_*` and `AWS_WEB_IDENTITY_TOKEN_FILE` variables. +Some models, such as OpenAI's GPT models, Grok, and Gemma on Bedrock, are only served by Bedrock Mantle, Bedrock's OpenAI-compatible endpoint. Pi sends them to Mantle's Responses API in your region. Mantle models currently need a Bedrock API key (`AWS_BEARER_TOKEN_BEDROCK` or a bearer token from `/login amazon-bedrock`); profiles and IAM credentials only work for the other Bedrock models. + ### Cloudflare AI Gateway The gateway requires a token, account ID, and gateway ID: diff --git a/scripts/model-catalog-minimum-version.mjs b/scripts/model-catalog-minimum-version.mjs index 6b3a49621..19f3fec18 100644 --- a/scripts/model-catalog-minimum-version.mjs +++ b/scripts/model-catalog-minimum-version.mjs @@ -7,5 +7,6 @@ // 1.2.1: kimi-coding models use openai-completions at /coding/v1; older clients only // implement anthropic-messages for kimi-coding and ignore a model's api. // 1.4.1: google-vertex serves Claude through anthropic-messages; older clients only -// implement Google's Vertex API for google-vertex. +// implement Google's Vertex API for google-vertex. amazon-bedrock serves Mantle-only +// models through openai-responses; older clients only implement Converse for it. export const MINIMUM_PI_VERSION = "1.4.1";