feat: record response, tool execution, and durable task times

pi-ai's AssistantMessageEventStream sets AssistantMessage.durationMs with a
monotonic clock for every API implementation. The agent loop and the durable
tool task set ToolResultMessage.durationMs around execute(). Durable task
records carry startedAt/endedAt stamped by the Session. The coding agent's bash
renderer shows Took from the recorded duration, also after a reload.

closes #10549
This commit is contained in:
Mario Zechner 2026-10-06 19:28:54 +02:00
parent 636703a0a4
commit 36a686ee8d
30 changed files with 451 additions and 22 deletions

View file

@ -2,6 +2,10 @@
## [Unreleased]
### Added
- Added `durationMs` to tool result messages, `AgentToolCallOutcome`, and `tool_execution_end` events: how long `execute()` took, measured with a monotonic clock and excluding hooks. Calls that did not run have none ([#10549](https://github.com/earendil-works/pi/issues/10549))
## [1.0.4] - 2026-10-05
## [1.0.3] - 2026-10-05

View file

@ -675,6 +675,7 @@ type ImmediateToolCallOutcome = {
type ExecutedToolCallOutcome = {
result: AgentToolResult<any>;
isError: boolean;
durationMs: number;
};
type FinalizedToolCallOutcome = AgentToolCallOutcome;
@ -824,6 +825,8 @@ async function executePreparedToolCall(
): Promise<ExecutedToolCallOutcome> {
const updateEvents: Promise<void>[] = [];
let acceptingUpdates = true;
const startedAt = performance.now();
const elapsed = () => Math.round(performance.now() - startedAt);
try {
const result = await prepared.tool.execute(
@ -835,15 +838,18 @@ async function executePreparedToolCall(
updateEvents.push(Promise.resolve(onUpdate(partialResult)));
},
);
const durationMs = elapsed();
acceptingUpdates = false;
await Promise.all(updateEvents);
return { result, isError: result.isError === true };
return { result, isError: result.isError === true, durationMs };
} catch (error) {
const durationMs = elapsed();
acceptingUpdates = false;
await Promise.all(updateEvents);
return {
result: createErrorToolResult(error instanceof Error ? error.message : String(error)),
isError: true,
durationMs,
};
} finally {
acceptingUpdates = false;
@ -899,6 +905,7 @@ async function finalizeExecutedToolCall(
toolCall: prepared.toolCall,
result,
isError,
durationMs: executed.durationMs,
};
}
@ -916,6 +923,7 @@ async function emitToolExecutionEnd(finalized: FinalizedToolCallOutcome, emit: A
toolName: finalized.toolCall.name,
result: finalized.result,
isError: finalized.isError,
...(finalized.durationMs === undefined ? {} : { durationMs: finalized.durationMs }),
});
}
@ -930,6 +938,7 @@ function createToolResultMessage(finalized: FinalizedToolCallOutcome): ToolResul
details: finalized.result.details,
usage: finalized.result.usage,
isError: finalized.isError,
...(finalized.durationMs === undefined ? {} : { durationMs: finalized.durationMs }),
timestamp: Date.now(),
};
}

View file

@ -450,6 +450,8 @@ export interface AgentToolCallOutcome {
toolCall: AgentToolCall;
result: AgentToolResult<any>;
isError: boolean;
/** Milliseconds `execute()` took, measured with a monotonic clock; absent when the tool did not run. */
durationMs?: number;
}
/**
@ -526,4 +528,12 @@ export type AgentEvent =
// Tool execution lifecycle
| { type: "tool_execution_start"; toolCallId: string; toolName: string; args: any }
| { type: "tool_execution_update"; toolCallId: string; toolName: string; args: any; partialResult: any }
| { type: "tool_execution_end"; toolCallId: string; toolName: string; result: any; isError: boolean };
| {
type: "tool_execution_end";
toolCallId: string;
toolName: string;
result: any;
isError: boolean;
/** Milliseconds `execute()` took, measured with a monotonic clock; absent when the tool did not run. */
durationMs?: number;
};

View file

@ -4,6 +4,7 @@ import {
EventStream,
type Message,
type Model,
type ToolResultMessage,
type UserMessage,
} from "@earendil-works/pi-ai";
import { Type } from "typebox";
@ -412,6 +413,61 @@ describe("agentLoop with AgentMessage", () => {
expect(toolResult?.role === "toolResult" ? toolResult.usage : undefined).toEqual(patchedToolUsage);
});
// #10549
it("records how long execute() took on the tool result, excluding hooks", async () => {
const toolSchema = Type.Object({ value: Type.String() });
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
const tool: AgentTool<typeof toolSchema, { value: string }> = {
name: "echo",
label: "Echo",
description: "Echo tool",
parameters: toolSchema,
async execute(_toolCallId, params) {
await sleep(30);
return { content: [{ type: "text", text: params.value }], details: { value: params.value } };
},
};
const config: AgentLoopConfig = {
model: createModel(),
convertToLlm: identityConverter,
beforeToolCall: async ({ toolCall }) => {
await sleep(100);
return toolCall.id === "blocked" ? { block: true, reason: "no" } : undefined;
},
};
let callIndex = 0;
const streamFn = () => {
const stream = new MockAssistantStream();
queueMicrotask(() => {
const message =
callIndex === 0
? createAssistantMessage(
[
{ type: "toolCall", id: "ran", name: "echo", arguments: { value: "a" } },
{ type: "toolCall", id: "blocked", name: "echo", arguments: { value: "b" } },
],
"toolUse",
)
: createAssistantMessage([{ type: "text", text: "done" }]);
stream.push({ type: "done", reason: callIndex === 0 ? "toolUse" : "stop", message });
callIndex++;
});
return stream;
};
const stream = agentLoop([createUserMessage("go")], { messages: [], tools: [tool] }, config, undefined, streamFn);
for await (const _event of stream) {
// drain
}
const results = (await stream.result()).filter(
(message): message is ToolResultMessage => message.role === "toolResult",
);
const [ran, blocked] = results;
expect(ran?.durationMs).toBeGreaterThanOrEqual(25);
expect(ran?.durationMs).toBeLessThan(100);
expect(blocked?.isError).toBe(true);
expect(blocked).not.toHaveProperty("durationMs");
});
it("should not execute tool calls from a length-truncated assistant message", async () => {
const toolSchema = Type.Object({ value: Type.String() });
const executed: string[] = [];