From 23477ed6acad0c201a38462256fc43d95f02e4f2 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Sat, 24 Oct 2026 17:41:50 +0200 Subject: [PATCH] feat(ai): serve Claude models on Google Vertex AI google-vertex now lists the Claude models models.dev catalogs on Vertex's Anthropic Messages API. A thin wrapper reuses the Anthropic Messages implementation: it rewrites /v1/messages to the per-model streamRawPredict and rawPredict endpoints, moves the model into the URL, adds anthropic_version, and authenticates with an Application Default Credentials access token. google-auth-library becomes a direct dependency (already in the tree through @google/genai) and is stubbed for browser bundles. --- package-lock.json | 1 + packages/ai/README.md | 6 +- .../ai/catalog/providers/google-vertex.ts | 17 ++- packages/ai/package.json | 4 + .../src/api/google-vertex-anthropic.lazy.ts | 4 + .../ai/src/api/google-vertex-anthropic.ts | 135 ++++++++++++++++++ packages/ai/src/providers/google-vertex.ts | 11 +- .../ai/test/google-vertex-anthropic.test.ts | 114 +++++++++++++++ packages/ai/test/stream.test.ts | 23 +++ packages/coding-agent/docs/providers.md | 2 + .../install-lock/package-lock.json | 1 + scripts/model-catalog-minimum-version.mjs | 4 +- 12 files changed, 313 insertions(+), 9 deletions(-) create mode 100644 packages/ai/src/api/google-vertex-anthropic.lazy.ts create mode 100644 packages/ai/src/api/google-vertex-anthropic.ts create mode 100644 packages/ai/test/google-vertex-anthropic.test.ts diff --git a/package-lock.json b/package-lock.json index 74451ff75..d24542930 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5729,6 +5729,7 @@ "@earendil-works/pi-telemetry": "1.4.0", "@google/genai": "2.21.0", "@smithy/node-http-handler": "4.12.1", + "google-auth-library": "10.6.2", "http-proxy-agent": "9.1.0", "https-proxy-agent": "9.1.0", "openai": "7.19.0", diff --git a/packages/ai/README.md b/packages/ai/README.md index f2c75f976..3718ef092 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -71,7 +71,7 @@ Unified LLM API with provider collections, automatic auth resolution, token and - **NVIDIA NIM** - **Anthropic** - **Google** -- **Vertex AI** (Gemini via Vertex AI) +- **Vertex AI** (Gemini, and Claude through Anthropic's Messages API on Vertex) - **Mistral** - **Groq** - **Cerebras** @@ -1874,7 +1874,9 @@ export GOOGLE_CLOUD_LOCATION="us-central1" export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json" ``` -Official docs: [Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials) +Claude models on Vertex (for example `claude-opus-4-8@default`) use Anthropic's Messages API at Vertex's `rawPredict` endpoints. They need ADC or a service account plus a project and location; Vertex API keys only work for Gemini. Enable each Claude model in the Vertex AI Model Garden first, and use a location that serves it (for example `us-east5` or `global`). + +Official docs: [Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials), [Claude on Vertex AI](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/use-claude) ### CLI Login diff --git a/packages/ai/catalog/providers/google-vertex.ts b/packages/ai/catalog/providers/google-vertex.ts index e74a4ed5f..df4449af0 100644 --- a/packages/ai/catalog/providers/google-vertex.ts +++ b/packages/ai/catalog/providers/google-vertex.ts @@ -3,18 +3,21 @@ import { googleThinkingLevels } from "../families/google.ts"; import { modelsDevChatModels } from "../sources/models-dev.ts"; import { GOOGLE_ALIAS_TARGETS } from "./google.ts"; +const BASE_URL = "https://{location}-aiplatform.googleapis.com"; + export default defineProvider({ id: "google-vertex", models: (ctx) => { const sources = new Map( modelsDevChatModels(ctx.sources.modelsDev, "google-vertex").map((model) => [model.id, model]), ); - return modelsDevModels(ctx, "google-vertex", { + const gemini = modelsDevModels(ctx, "google-vertex", { api: "google-vertex", - baseUrl: "https://{location}-aiplatform.googleapis.com", + baseUrl: BASE_URL, recordReasoning: false, // The models.dev catalog also includes Claude, OpenAI, and other MaaS models - // that do not use the @google/genai Gemini streaming path. + // that do not use the @google/genai Gemini streaming path. Claude is added + // below; the other partner models are not supported yet. include: (source) => source.id.startsWith("gemini-") && source.id !== "gemini-3.1-flash-lite-preview", map: (model, aliasSource) => { const target = GOOGLE_ALIAS_TARGETS[model.id]; @@ -39,5 +42,13 @@ export default defineProvider({ }; }, }); + // Claude through Anthropic's Messages API on Vertex (`rawPredict` endpoints). + const claude = modelsDevModels(ctx, "google-vertex", { + api: "anthropic-messages", + baseUrl: BASE_URL, + skipDeprecated: true, + protocol: "messages", + }); + return [...gemini, ...claude]; }, }); diff --git a/packages/ai/package.json b/packages/ai/package.json index 6e04980f0..eb399112e 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -51,6 +51,9 @@ "bin": { "pi-ai": "dist/cli.js" }, + "browser": { + "google-auth-library": false + }, "files": [ "dist", "README.md" @@ -72,6 +75,7 @@ "@earendil-works/pi-telemetry": "1.4.0", "@google/genai": "2.21.0", "@smithy/node-http-handler": "4.12.1", + "google-auth-library": "10.6.2", "http-proxy-agent": "9.1.0", "https-proxy-agent": "9.1.0", "openai": "7.19.0", diff --git a/packages/ai/src/api/google-vertex-anthropic.lazy.ts b/packages/ai/src/api/google-vertex-anthropic.lazy.ts new file mode 100644 index 000000000..e6633e689 --- /dev/null +++ b/packages/ai/src/api/google-vertex-anthropic.lazy.ts @@ -0,0 +1,4 @@ +import type { ProviderStreams } from "../types.ts"; +import { lazyApi } from "./lazy.ts"; + +export const googleVertexAnthropicApi = (): ProviderStreams => lazyApi(() => import("./google-vertex-anthropic.ts")); diff --git a/packages/ai/src/api/google-vertex-anthropic.ts b/packages/ai/src/api/google-vertex-anthropic.ts new file mode 100644 index 000000000..393a4e381 --- /dev/null +++ b/packages/ai/src/api/google-vertex-anthropic.ts @@ -0,0 +1,135 @@ +// Claude on Vertex AI. Vertex serves Anthropic's Messages API at per-model +// `rawPredict` endpoints with Google OAuth instead of an Anthropic key, so this +// module reuses the Anthropic Messages implementation and rewrites its requests: +// the model moves from the body into the URL, and the body names the Vertex +// API version. +// https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/claude/use-claude + +import { GoogleAuth } from "google-auth-library"; +import type { Model, ProviderEnv, SimpleStreamOptions, StreamFunction, TranscriptContext } from "../types.ts"; +import type { AssistantMessageEventStream } from "../utils/event-stream.ts"; +import { getProviderEnvValue } from "../utils/provider-env.ts"; +import { + type AnthropicOptions, + stream as anthropicStream, + streamSimple as anthropicStreamSimple, +} from "./anthropic-messages.ts"; +import { lazyStream } from "./lazy.ts"; + +export const VERTEX_ANTHROPIC_VERSION = "vertex-2023-10-16"; +const CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform"; + +export interface VertexAnthropicTarget { + project: string; + location: string; +} + +/** The Vertex host for a location. The `global` location has no regional prefix. */ +export function vertexHost(location: string): string { + return location === "global" ? "aiplatform.googleapis.com" : `${location}-aiplatform.googleapis.com`; +} + +/** + * Wrap `fetch` so Anthropic SDK requests reach Vertex: `/v1/messages` becomes + * `…/publishers/anthropic/models/{model}:streamRawPredict` (or `rawPredict`), + * token counting becomes `count-tokens:rawPredict`, and the Google access token + * replaces Anthropic's key header. Other requests pass through unchanged. + */ +export function createVertexAnthropicFetch( + target: VertexAnthropicTarget, + accessToken: string, + baseFetch: typeof globalThis.fetch = globalThis.fetch, +): typeof globalThis.fetch { + const modelsPath = `/v1/projects/${target.project}/locations/${target.location}/publishers/anthropic/models`; + return async (input, init) => { + const url = new URL(typeof input === "string" || input instanceof URL ? input : input.url); + const endpoint = /\/v1\/messages(\/count_tokens)?$/.exec(url.pathname); + if (!endpoint || typeof init?.body !== "string") return baseFetch(input, init); + + const { model, ...body } = JSON.parse(init.body) as { model?: string; stream?: boolean }; + const path = endpoint[1] + ? `${modelsPath}/count-tokens:rawPredict` + : `${modelsPath}/${model}:${body.stream ? "streamRawPredict" : "rawPredict"}`; + const headers = new Headers(init.headers); + headers.delete("x-api-key"); + headers.set("authorization", `Bearer ${accessToken}`); + return baseFetch(`https://${vertexHost(target.location)}${path}`, { + ...init, + headers, + // Token counting keeps the model in the body. + body: JSON.stringify({ + anthropic_version: VERTEX_ANTHROPIC_VERSION, + ...(endpoint[1] ? { model } : {}), + ...body, + }), + }); + }; +} + +function resolveTarget(env: ProviderEnv | undefined): VertexAnthropicTarget { + const project = getProviderEnvValue("GOOGLE_CLOUD_PROJECT", env) || getProviderEnvValue("GCLOUD_PROJECT", env); + if (!project) { + throw new Error("Claude on Vertex AI requires a project ID. Set GOOGLE_CLOUD_PROJECT or GCLOUD_PROJECT."); + } + const location = getProviderEnvValue("GOOGLE_CLOUD_LOCATION", env); + if (!location) throw new Error("Claude on Vertex AI requires a location. Set GOOGLE_CLOUD_LOCATION."); + return { project, location }; +} + +// One GoogleAuth per credentials file, so access tokens are cached and refreshed. +const authClients = new Map(); + +async function getAccessToken(env: ProviderEnv | undefined): Promise { + const keyFilename = getProviderEnvValue("GOOGLE_APPLICATION_CREDENTIALS", env); + let auth = authClients.get(keyFilename ?? ""); + if (!auth) { + // Browser bundles replace google-auth-library with an empty module. + if (typeof GoogleAuth !== "function") throw new Error("Claude on Vertex AI requires Node.js or Bun."); + auth = new GoogleAuth({ scopes: CLOUD_PLATFORM_SCOPE, ...(keyFilename ? { keyFilename } : {}) }); + authClients.set(keyFilename ?? "", auth); + } + const token = await auth.getAccessToken(); + if (!token) { + // Vertex API keys only work for Google's own models. + throw new Error( + "Claude on Vertex AI requires Application Default Credentials or a service account; API keys are not supported.", + ); + } + return token; +} + +async function prepare( + model: Model<"anthropic-messages">, + options: TOptions | undefined, +): Promise<{ model: Model<"anthropic-messages">; options: TOptions }> { + const target = resolveTarget(options?.env); + const accessToken = await getAccessToken(options?.env); + return { + model: { ...model, baseUrl: `https://${vertexHost(target.location)}` }, + options: { + ...options, + apiKey: accessToken, + fetch: createVertexAnthropicFetch(target, accessToken, options?.fetch), + } as TOptions, + }; +} + +export const stream: StreamFunction<"anthropic-messages", AnthropicOptions> = ( + model: Model<"anthropic-messages">, + context: TranscriptContext, + options?: AnthropicOptions, +): AssistantMessageEventStream => + lazyStream(model, async () => { + const prepared = await prepare(model, options); + return anthropicStream(prepared.model, context, prepared.options); + }); + +export const streamSimple: StreamFunction<"anthropic-messages", SimpleStreamOptions> = ( + model: Model<"anthropic-messages">, + context: TranscriptContext, + options?: SimpleStreamOptions, +): AssistantMessageEventStream => + lazyStream(model, async () => { + const prepared = await prepare(model, options); + return anthropicStreamSimple(prepared.model, context, prepared.options); + }); diff --git a/packages/ai/src/providers/google-vertex.ts b/packages/ai/src/providers/google-vertex.ts index 83c7d0fe8..97b5dc73f 100644 --- a/packages/ai/src/providers/google-vertex.ts +++ b/packages/ai/src/providers/google-vertex.ts @@ -1,4 +1,5 @@ import { googleVertexApi } from "../api/google-vertex.lazy.ts"; +import { googleVertexAnthropicApi } from "../api/google-vertex-anthropic.lazy.ts"; import type { ApiKeyAuth } from "../auth/types.ts"; import { createProvider, type Provider } from "../models.ts"; import { GOOGLE_VERTEX_MODELS } from "./google-vertex.models.ts"; @@ -89,12 +90,16 @@ const vertexAuth: ApiKeyAuth = { }, }; -export function googleVertexProvider(): Provider<"google-vertex"> { - return createProvider({ +/** Gemini through the Gen AI SDK, and Claude through Anthropic's Messages API on Vertex. */ +export function googleVertexProvider(): Provider<"google-vertex" | "anthropic-messages"> { + return createProvider<"google-vertex" | "anthropic-messages">({ id: "google-vertex", name: "Google Vertex AI", auth: { apiKey: vertexAuth }, models: Object.values(GOOGLE_VERTEX_MODELS), - api: googleVertexApi(), + api: { + "google-vertex": googleVertexApi(), + "anthropic-messages": googleVertexAnthropicApi(), + }, }); } diff --git a/packages/ai/test/google-vertex-anthropic.test.ts b/packages/ai/test/google-vertex-anthropic.test.ts new file mode 100644 index 000000000..9a80bb923 --- /dev/null +++ b/packages/ai/test/google-vertex-anthropic.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from "vitest"; +import { streamSimple as anthropicStreamSimple } from "../src/api/anthropic-messages.ts"; +import { + createVertexAnthropicFetch, + VERTEX_ANTHROPIC_VERSION, + vertexHost, +} from "../src/api/google-vertex-anthropic.ts"; +import { hasApi } from "../src/models.ts"; +import { getBuiltinModels } from "../src/providers/all.ts"; +import type { Model } from "../src/types.ts"; +import { normalizeContext } from "../src/utils/transcript.ts"; + +interface CapturedRequest { + url: string; + headers: Headers; + body: Record; +} + +function capturingFetch(captured: CapturedRequest[]): typeof globalThis.fetch { + return async (input, init) => { + captured.push({ + url: String(input), + headers: new Headers(init?.headers), + body: JSON.parse(String(init?.body)), + }); + return new Response(JSON.stringify({ error: { type: "test", message: "captured" } }), { status: 400 }); + }; +} + +const target = { project: "my-project", location: "us-east5" }; + +describe("Claude on Vertex AI transport", () => { + it("rewrites streaming Messages requests to streamRawPredict", async () => { + const captured: CapturedRequest[] = []; + const vertexFetch = createVertexAnthropicFetch(target, "ya29.token", capturingFetch(captured)); + await vertexFetch("https://us-east5-aiplatform.googleapis.com/v1/messages?beta=true", { + method: "POST", + headers: { "x-api-key": "ya29.token", "anthropic-beta": "some-beta" }, + body: JSON.stringify({ model: "claude-opus-4-8@default", stream: true, max_tokens: 10, messages: [] }), + }); + + expect(captured[0].url).toBe( + "https://us-east5-aiplatform.googleapis.com/v1/projects/my-project/locations/us-east5/publishers/anthropic/models/claude-opus-4-8@default:streamRawPredict", + ); + expect(captured[0].headers.get("authorization")).toBe("Bearer ya29.token"); + expect(captured[0].headers.has("x-api-key")).toBe(false); + expect(captured[0].headers.get("anthropic-beta")).toBe("some-beta"); + expect(captured[0].body).toEqual({ + anthropic_version: VERTEX_ANTHROPIC_VERSION, + stream: true, + max_tokens: 10, + messages: [], + }); + }); + + it("uses rawPredict for non-streaming requests and keeps the model for token counting", async () => { + const captured: CapturedRequest[] = []; + const vertexFetch = createVertexAnthropicFetch( + { project: "p", location: "global" }, + "t", + capturingFetch(captured), + ); + await vertexFetch("https://aiplatform.googleapis.com/v1/messages", { + method: "POST", + body: JSON.stringify({ model: "claude-haiku-4-5@20251001", messages: [] }), + }); + await vertexFetch("https://aiplatform.googleapis.com/v1/messages/count_tokens", { + method: "POST", + body: JSON.stringify({ model: "claude-haiku-4-5@20251001", messages: [] }), + }); + + expect(captured[0].url).toBe( + "https://aiplatform.googleapis.com/v1/projects/p/locations/global/publishers/anthropic/models/claude-haiku-4-5@20251001:rawPredict", + ); + expect(captured[0].body).not.toHaveProperty("model"); + expect(captured[1].url).toBe( + "https://aiplatform.googleapis.com/v1/projects/p/locations/global/publishers/anthropic/models/count-tokens:rawPredict", + ); + expect(captured[1].body).toMatchObject({ model: "claude-haiku-4-5@20251001" }); + }); + + it("maps the global location to the unprefixed host", () => { + expect(vertexHost("global")).toBe("aiplatform.googleapis.com"); + expect(vertexHost("europe-west1")).toBe("europe-west1-aiplatform.googleapis.com"); + }); + + it("routes Anthropic SDK requests through the Vertex endpoint", async () => { + const model = getBuiltinModels("google-vertex") + .filter((candidate) => hasApi(candidate, "anthropic-messages")) + .find((candidate) => candidate.id === "claude-opus-4-8@default") as Model<"anthropic-messages"> | undefined; + if (!model) throw new Error("Vertex catalog has no claude-opus-4-8@default"); + const captured: CapturedRequest[] = []; + const result = await anthropicStreamSimple( + { ...model, baseUrl: `https://${vertexHost(target.location)}` }, + normalizeContext({ messages: [{ role: "user", content: "hi", timestamp: 0 }] }), + { apiKey: "ya29.token", fetch: createVertexAnthropicFetch(target, "ya29.token", capturingFetch(captured)) }, + ).result(); + + expect(result.stopReason).toBe("error"); + expect(captured).toHaveLength(1); + expect(captured[0].url).toBe( + "https://us-east5-aiplatform.googleapis.com/v1/projects/my-project/locations/us-east5/publishers/anthropic/models/claude-opus-4-8@default:streamRawPredict", + ); + expect(captured[0].body).toMatchObject({ anthropic_version: VERTEX_ANTHROPIC_VERSION, stream: true }); + expect(captured[0].body).not.toHaveProperty("model"); + expect(captured[0].headers.get("authorization")).toBe("Bearer ya29.token"); + }); + + it("lists Claude models on google-vertex through Anthropic Messages", () => { + const claude = getBuiltinModels("google-vertex").filter((model) => model.id.startsWith("claude-")); + expect(claude.length).toBeGreaterThan(0); + for (const model of claude) expect(model.api).toBe("anthropic-messages"); + }); +}); diff --git a/packages/ai/test/stream.test.ts b/packages/ai/test/stream.test.ts index df2644d4c..7e763b14c 100644 --- a/packages/ai/test/stream.test.ts +++ b/packages/ai/test/stream.test.ts @@ -421,6 +421,29 @@ describe("Generate E2E Tests", () => { }); }); + describe.skipIf( + !(process.env.GOOGLE_CLOUD_PROJECT || process.env.GCLOUD_PROJECT) || !process.env.GOOGLE_CLOUD_LOCATION, + )("Google Vertex Provider (claude-haiku-4-5 via Anthropic Messages)", () => { + // Claude on Vertex authenticates with Application Default Credentials only. + const model = getModel("google-vertex", "claude-haiku-4-5@20251001"); + + it("should complete basic text generation", { retry: 3 }, async () => { + await basicTextGeneration(model, { thinkingEnabled: true }); + }); + + it("should handle tool calling", { retry: 3 }, async () => { + await handleToolCall(model); + }); + + it("should handle streaming", { retry: 3 }, async () => { + await handleStreaming(model); + }); + + it("should handle image input", { retry: 3 }, async () => { + await handleImage(model); + }); + }); + describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Completions Provider (gpt-4o-mini)", () => { const { compat: _compat, ...baseModel } = getModel("openai", "gpt-4o-mini"); void _compat; diff --git a/packages/coding-agent/docs/providers.md b/packages/coding-agent/docs/providers.md index 4d9ac8439..7b56dc46e 100644 --- a/packages/coding-agent/docs/providers.md +++ b/packages/coding-agent/docs/providers.md @@ -261,3 +261,5 @@ gcloud auth application-default login ``` To use a service-account key file instead, set `GOOGLE_APPLICATION_CREDENTIALS` along with the project and location. + +Claude models on Vertex, such as `google-vertex/claude-opus-4-8@default`, need Application Default Credentials or a service account; a Google Cloud API key only works for Gemini. Enable the Claude model in the Vertex AI Model Garden and set `GOOGLE_CLOUD_LOCATION` to a location that serves it, such as `us-east5` or `global`. diff --git a/packages/coding-agent/install-lock/package-lock.json b/packages/coding-agent/install-lock/package-lock.json index 25e273cf6..bc6c6bb41 100644 --- a/packages/coding-agent/install-lock/package-lock.json +++ b/packages/coding-agent/install-lock/package-lock.json @@ -407,6 +407,7 @@ "@earendil-works/pi-telemetry": "1.4.0", "@google/genai": "2.21.0", "@smithy/node-http-handler": "4.12.1", + "google-auth-library": "10.6.2", "http-proxy-agent": "9.1.0", "https-proxy-agent": "9.1.0", "openai": "7.19.0", diff --git a/scripts/model-catalog-minimum-version.mjs b/scripts/model-catalog-minimum-version.mjs index d7d5068cf..6b3a49621 100644 --- a/scripts/model-catalog-minimum-version.mjs +++ b/scripts/model-catalog-minimum-version.mjs @@ -6,4 +6,6 @@ // older pi clients, in the same commit as the change that requires it. // 1.2.1: kimi-coding models use openai-completions at /coding/v1; older clients only // implement anthropic-messages for kimi-coding and ignore a model's api. -export const MINIMUM_PI_VERSION = "1.2.1"; +// 1.4.1: google-vertex serves Claude through anthropic-messages; older clients only +// implement Google's Vertex API for google-vertex. +export const MINIMUM_PI_VERSION = "1.4.1";