mirror of
https://github.com/badlogic/pi-mono.git
synced 2026-11-05 23:37:25 +00:00
feat: record response, tool execution, and durable task times
pi-ai's AssistantMessageEventStream sets AssistantMessage.durationMs with a monotonic clock for every API implementation. The agent loop and the durable tool task set ToolResultMessage.durationMs around execute(). Durable task records carry startedAt/endedAt stamped by the Session. The coding agent's bash renderer shows Took from the recorded duration, also after a reload. closes #10549
This commit is contained in:
parent
636703a0a4
commit
36a686ee8d
30 changed files with 451 additions and 22 deletions
|
|
@ -2,6 +2,10 @@
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- Added `durationMs` to tool result messages, `AgentToolCallOutcome`, and `tool_execution_end` events: how long `execute()` took, measured with a monotonic clock and excluding hooks. Calls that did not run have none ([#10549](https://github.com/earendil-works/pi/issues/10549))
|
||||
|
||||
## [1.0.4] - 2026-10-05
|
||||
|
||||
## [1.0.3] - 2026-10-05
|
||||
|
|
|
|||
|
|
@ -675,6 +675,7 @@ type ImmediateToolCallOutcome = {
|
|||
type ExecutedToolCallOutcome = {
|
||||
result: AgentToolResult<any>;
|
||||
isError: boolean;
|
||||
durationMs: number;
|
||||
};
|
||||
|
||||
type FinalizedToolCallOutcome = AgentToolCallOutcome;
|
||||
|
|
@ -824,6 +825,8 @@ async function executePreparedToolCall(
|
|||
): Promise<ExecutedToolCallOutcome> {
|
||||
const updateEvents: Promise<void>[] = [];
|
||||
let acceptingUpdates = true;
|
||||
const startedAt = performance.now();
|
||||
const elapsed = () => Math.round(performance.now() - startedAt);
|
||||
|
||||
try {
|
||||
const result = await prepared.tool.execute(
|
||||
|
|
@ -835,15 +838,18 @@ async function executePreparedToolCall(
|
|||
updateEvents.push(Promise.resolve(onUpdate(partialResult)));
|
||||
},
|
||||
);
|
||||
const durationMs = elapsed();
|
||||
acceptingUpdates = false;
|
||||
await Promise.all(updateEvents);
|
||||
return { result, isError: result.isError === true };
|
||||
return { result, isError: result.isError === true, durationMs };
|
||||
} catch (error) {
|
||||
const durationMs = elapsed();
|
||||
acceptingUpdates = false;
|
||||
await Promise.all(updateEvents);
|
||||
return {
|
||||
result: createErrorToolResult(error instanceof Error ? error.message : String(error)),
|
||||
isError: true,
|
||||
durationMs,
|
||||
};
|
||||
} finally {
|
||||
acceptingUpdates = false;
|
||||
|
|
@ -899,6 +905,7 @@ async function finalizeExecutedToolCall(
|
|||
toolCall: prepared.toolCall,
|
||||
result,
|
||||
isError,
|
||||
durationMs: executed.durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -916,6 +923,7 @@ async function emitToolExecutionEnd(finalized: FinalizedToolCallOutcome, emit: A
|
|||
toolName: finalized.toolCall.name,
|
||||
result: finalized.result,
|
||||
isError: finalized.isError,
|
||||
...(finalized.durationMs === undefined ? {} : { durationMs: finalized.durationMs }),
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -930,6 +938,7 @@ function createToolResultMessage(finalized: FinalizedToolCallOutcome): ToolResul
|
|||
details: finalized.result.details,
|
||||
usage: finalized.result.usage,
|
||||
isError: finalized.isError,
|
||||
...(finalized.durationMs === undefined ? {} : { durationMs: finalized.durationMs }),
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -450,6 +450,8 @@ export interface AgentToolCallOutcome {
|
|||
toolCall: AgentToolCall;
|
||||
result: AgentToolResult<any>;
|
||||
isError: boolean;
|
||||
/** Milliseconds `execute()` took, measured with a monotonic clock; absent when the tool did not run. */
|
||||
durationMs?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -526,4 +528,12 @@ export type AgentEvent =
|
|||
// Tool execution lifecycle
|
||||
| { type: "tool_execution_start"; toolCallId: string; toolName: string; args: any }
|
||||
| { type: "tool_execution_update"; toolCallId: string; toolName: string; args: any; partialResult: any }
|
||||
| { type: "tool_execution_end"; toolCallId: string; toolName: string; result: any; isError: boolean };
|
||||
| {
|
||||
type: "tool_execution_end";
|
||||
toolCallId: string;
|
||||
toolName: string;
|
||||
result: any;
|
||||
isError: boolean;
|
||||
/** Milliseconds `execute()` took, measured with a monotonic clock; absent when the tool did not run. */
|
||||
durationMs?: number;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import {
|
|||
EventStream,
|
||||
type Message,
|
||||
type Model,
|
||||
type ToolResultMessage,
|
||||
type UserMessage,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import { Type } from "typebox";
|
||||
|
|
@ -412,6 +413,61 @@ describe("agentLoop with AgentMessage", () => {
|
|||
expect(toolResult?.role === "toolResult" ? toolResult.usage : undefined).toEqual(patchedToolUsage);
|
||||
});
|
||||
|
||||
// #10549
|
||||
it("records how long execute() took on the tool result, excluding hooks", async () => {
|
||||
const toolSchema = Type.Object({ value: Type.String() });
|
||||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
const tool: AgentTool<typeof toolSchema, { value: string }> = {
|
||||
name: "echo",
|
||||
label: "Echo",
|
||||
description: "Echo tool",
|
||||
parameters: toolSchema,
|
||||
async execute(_toolCallId, params) {
|
||||
await sleep(30);
|
||||
return { content: [{ type: "text", text: params.value }], details: { value: params.value } };
|
||||
},
|
||||
};
|
||||
const config: AgentLoopConfig = {
|
||||
model: createModel(),
|
||||
convertToLlm: identityConverter,
|
||||
beforeToolCall: async ({ toolCall }) => {
|
||||
await sleep(100);
|
||||
return toolCall.id === "blocked" ? { block: true, reason: "no" } : undefined;
|
||||
},
|
||||
};
|
||||
let callIndex = 0;
|
||||
const streamFn = () => {
|
||||
const stream = new MockAssistantStream();
|
||||
queueMicrotask(() => {
|
||||
const message =
|
||||
callIndex === 0
|
||||
? createAssistantMessage(
|
||||
[
|
||||
{ type: "toolCall", id: "ran", name: "echo", arguments: { value: "a" } },
|
||||
{ type: "toolCall", id: "blocked", name: "echo", arguments: { value: "b" } },
|
||||
],
|
||||
"toolUse",
|
||||
)
|
||||
: createAssistantMessage([{ type: "text", text: "done" }]);
|
||||
stream.push({ type: "done", reason: callIndex === 0 ? "toolUse" : "stop", message });
|
||||
callIndex++;
|
||||
});
|
||||
return stream;
|
||||
};
|
||||
const stream = agentLoop([createUserMessage("go")], { messages: [], tools: [tool] }, config, undefined, streamFn);
|
||||
for await (const _event of stream) {
|
||||
// drain
|
||||
}
|
||||
const results = (await stream.result()).filter(
|
||||
(message): message is ToolResultMessage => message.role === "toolResult",
|
||||
);
|
||||
const [ran, blocked] = results;
|
||||
expect(ran?.durationMs).toBeGreaterThanOrEqual(25);
|
||||
expect(ran?.durationMs).toBeLessThan(100);
|
||||
expect(blocked?.isError).toBe(true);
|
||||
expect(blocked).not.toHaveProperty("durationMs");
|
||||
});
|
||||
|
||||
it("should not execute tool calls from a length-truncated assistant message", async () => {
|
||||
const toolSchema = Type.Object({ value: Type.String() });
|
||||
const executed: string[] = [];
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue