sscoble/pi-mono
1
0
Fork 0
mirror of https://github.com/badlogic/pi-mono.git synced 2026-11-05 23:37:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

fix(ai): do not count faux cache writes as input tokens

Browse source 
With a sessionId, the faux provider reported newly cached prompt tokens
both as input and as cacheWrite, so input + cacheRead + cacheWrite
exceeded the estimated prompt. Every prompt token now goes either to
cacheRead or to cacheWrite, and input is 0 while caching is simulated.
This commit is contained in:
triagent 2026-10-22 23:34:02 +00:00 • committed by Petr Houška
parent 606af8662e
commit f3fa8bd924
2 changed files with 7 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

5
packages/ai/src/providers/faux.ts
View file

@ -263,11 +263,12 @@ function withUsageEstimate(
if (previousPrompt) {
const cachedChars = commonPromptPrefixLength(previousPrompt, prompt);
cacheRead = Math.ceil(cachedChars / 4);
cacheWrite = Math.ceil((promptLength - cachedChars) / 4);
input = Math.max(0, promptTokens - cacheRead);
cacheWrite = Math.max(0, promptTokens - cacheRead);
} else {
cacheWrite = promptTokens;
}
// Every prompt token is either read from or written to the simulated cache.
input = 0;
promptCache.set(sessionId, prompt);
}

7
packages/ai/test/faux-provider.test.ts
View file

@ -332,17 +332,18 @@ describe("faux provider", () => {
const user = (content: string) => ({ role: "user" as const, content, timestamp: 1 });
// Prompt texts: "user:hello world" (16 characters), then 16 + 2 + "user:next" (9) = 27.
await complete(registration.getModel(), { messages: [user("hello world")] }, options);
const first = await complete(registration.getModel(), { messages: [user("hello world")] }, options);
expect(first.usage).toMatchObject({ input: 0, cacheRead: 0, cacheWrite: 4 });
const extended = await complete(
registration.getModel(),
{ messages: [user("hello world"), user("next")] },
options,
);
expect(extended.usage).toMatchObject({ input: 3, cacheRead: 4, cacheWrite: 3 });
expect(extended.usage).toMatchObject({ input: 0, cacheRead: 4, cacheWrite: 3 });
// The first message now differs after "user:hello w" (12 characters): "user:hello wide" + 2 + 9 = 26.
const edited = await complete(registration.getModel(), { messages: [user("hello wide"), user("next")] }, options);
expect(edited.usage).toMatchObject({ input: 4, cacheRead: 3, cacheWrite: 4 });
expect(edited.usage).toMatchObject({ input: 0, cacheRead: 3, cacheWrite: 4 });
});
it("does not simulate caching when cacheRetention is none", async () => {