diff --git a/.github/APPROVED_CONTRIBUTORS b/.github/APPROVED_CONTRIBUTORS index a8991d355..4139119a3 100644 --- a/.github/APPROVED_CONTRIBUTORS +++ b/.github/APPROVED_CONTRIBUTORS @@ -391,3 +391,5 @@ edverma pr Charlie0113-T pr RedBeard0531 pr + +metaist pr diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 6f7664a28..594e6f93b 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -8,6 +8,7 @@ ### Fixed +- Fixed Amazon Bedrock one-hour cache writes being priced at the five-minute rate ([#9457](https://github.com/earendil-works/pi/issues/9457)). - Added `RetryPolicy.maxAgentDelayMs` support to cap shared assistant retry backoff for summarization calls ([#8826](https://github.com/earendil-works/pi/issues/8826)). - Fixed quadratic CPU usage when draining buffered `EventStream` events ([#9055](https://github.com/earendil-works/pi/issues/9055)). - Fixed Mistral Medium reasoning requests to use `reasoning_effort` for all reasoning-capable `mistral-medium-*` model IDs instead of the unsupported `prompt_mode` ([#8700](https://github.com/earendil-works/pi/issues/8700)). diff --git a/packages/ai/README.md b/packages/ai/README.md index 58e10acda..21b590ba9 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -1591,7 +1591,7 @@ Built-in login and refresh flows are private provider implementations. Use provi Provider notes: -**OpenAI Codex**: Requires a ChatGPT Plus or Pro subscription. Provides access to GPT-5.x Codex models with extended context windows and reasoning capabilities. The library automatically handles session-based prompt caching when `sessionId` is provided in stream options unless `cacheRetention` is `"none"`. You can set `transport` in stream options to `"sse"`, `"websocket"`, or `"auto"` for Codex Responses transport selection. When using WebSocket with a `sessionId` and cache retention enabled, connections are reused per session and expire after 5 minutes of inactivity. +**OpenAI Codex**: Requires a ChatGPT Plus or Pro subscription. Provides access to GPT-5.x Codex models with extended context windows and reasoning capabilities. The library automatically handles session-based prompt caching when `sessionId` is provided in stream options unless `cacheRetention` is `"none"`. You can set `transport` in stream options to `"sse"`, `"websocket"`, or `"auto"` for Codex Responses transport selection. When using WebSocket with a `sessionId` and cache retention enabled, connections are reused per session and expire after 5 minutes of inactivity. Call `cleanupSessionResources(sessionId)` when finished so the pooled connection does not keep the process alive. **Azure OpenAI (Responses)**: Uses the Responses API only. Set `AZURE_OPENAI_API_KEY` and either `AZURE_OPENAI_BASE_URL` or `AZURE_OPENAI_RESOURCE_NAME`. `AZURE_OPENAI_BASE_URL` supports both `https://.openai.azure.com` and `https://.cognitiveservices.azure.com`; root endpoints are normalized to `.../openai/v1` automatically. Use `AZURE_OPENAI_API_VERSION` (defaults to `v1`) to override the API version if needed. Deployment names are treated as model IDs by default, override with `azureDeploymentName` or `AZURE_OPENAI_DEPLOYMENT_NAME_MAP` using comma-separated `model-id=deployment` pairs (for example `gpt-4o-mini=my-deployment,gpt-4o=prod`). Legacy deployment-based URLs are intentionally unsupported. diff --git a/packages/ai/src/api/bedrock-converse-stream.ts b/packages/ai/src/api/bedrock-converse-stream.ts index 2e944fcd2..43a816e55 100644 --- a/packages/ai/src/api/bedrock-converse-stream.ts +++ b/packages/ai/src/api/bedrock-converse-stream.ts @@ -692,6 +692,10 @@ function handleMetadata( output.usage.output = event.usage.outputTokens || 0; output.usage.cacheRead = event.usage.cacheReadInputTokens || 0; output.usage.cacheWrite = event.usage.cacheWriteInputTokens || 0; + output.usage.cacheWrite1h = event.usage.cacheDetails?.reduce( + (total, detail) => total + (detail.ttl === CacheTTL.ONE_HOUR ? (detail.inputTokens ?? 0) : 0), + 0, + ); output.usage.totalTokens = event.usage.totalTokens || output.usage.input + output.usage.output; calculateCost(model, output.usage); } diff --git a/packages/ai/test/bedrock-cache-write-1h-cost.test.ts b/packages/ai/test/bedrock-cache-write-1h-cost.test.ts new file mode 100644 index 000000000..771b26b8f --- /dev/null +++ b/packages/ai/test/bedrock-cache-write-1h-cost.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@aws-sdk/client-bedrock-runtime", () => { + class BedrockRuntimeServiceException extends Error {} + + class BedrockRuntimeClient { + middlewareStack = { add: () => undefined }; + + async send(): Promise { + return { + $metadata: { httpStatusCode: 200 }, + stream: (async function* () { + yield { messageStart: { role: "assistant" } }; + yield { + metadata: { + usage: { + inputTokens: 100, + outputTokens: 5, + totalTokens: 1_000_105, + cacheWriteInputTokens: 1_000_000, + cacheDetails: [ + { ttl: "1h", inputTokens: 150_000 }, + { ttl: "5m", inputTokens: 600_000 }, + { ttl: "1h", inputTokens: 250_000 }, + ], + }, + }, + }; + yield { messageStop: { stopReason: "end_turn" } }; + })(), + }; + } + } + + class ConverseStreamCommand { + readonly input: unknown; + constructor(input: unknown) { + this.input = input; + } + } + + return { + BedrockRuntimeClient, + BedrockRuntimeServiceException, + ConverseStreamCommand, + StopReason: { + END_TURN: "end_turn", + STOP_SEQUENCE: "stop_sequence", + MAX_TOKENS: "max_tokens", + MODEL_CONTEXT_WINDOW_EXCEEDED: "model_context_window_exceeded", + TOOL_USE: "tool_use", + }, + CachePointType: { DEFAULT: "default" }, + CacheTTL: { FIVE_MINUTES: "5m", ONE_HOUR: "1h" }, + ConversationRole: { ASSISTANT: "assistant", USER: "user" }, + ImageFormat: { JPEG: "jpeg", PNG: "png", GIF: "gif", WEBP: "webp" }, + ToolResultStatus: { ERROR: "error", SUCCESS: "success" }, + }; +}); + +import { stream as streamBedrock } from "../src/api/bedrock-converse-stream.ts"; +import { getModel } from "../src/compat.ts"; +import type { Context } from "../src/types.ts"; + +const model = getModel("amazon-bedrock", "us.anthropic.claude-opus-4-8"); +const context: Context = { messages: [{ role: "user", content: "hi", timestamp: Date.now() }] }; + +describe("Bedrock 1h cache write cost", () => { + it("prices the 1h cache details at 2x while preserving the total cache write", async () => { + // Regression test for https://github.com/earendil-works/pi/issues/9457 + const result = await streamBedrock(model, context, { cacheRetention: "none" }).result(); + + expect(result.usage.cacheWrite).toBe(1_000_000); + expect(result.usage.cacheWrite1h).toBe(400_000); + // 600k * 6.25/Mtok + 400k * 10/Mtok = 3.75 + 4.0 = 7.75 + expect(result.usage.cost.cacheWrite).toBeCloseTo(7.75, 10); + }); +});