From 66fbdb1c59e1a46da76e18b0f3200d01f7ec2393 Mon Sep 17 00:00:00 2001 From: Christian Klotz Date: Sat, 25 Jul 2026 23:09:22 +0300 Subject: [PATCH] feat(coding-agent): add extension creation eval --- biome.json | 1 + package-lock.json | 1 + packages/evals/README.md | 9 +- packages/evals/package.json | 1 + packages/evals/src/extensions.eval.ts | 49 + packages/evals/src/pi-agent.ts | 137 ++ packages/evals/src/pi-harness.ts | 2082 +++++++++++++++-- ...eneral-knowledge.eval.ts => smoke.eval.ts} | 11 +- 8 files changed, 2131 insertions(+), 160 deletions(-) create mode 100644 packages/evals/src/extensions.eval.ts create mode 100644 packages/evals/src/pi-agent.ts rename packages/evals/src/{general-knowledge.eval.ts => smoke.eval.ts} (54%) diff --git a/biome.json b/biome.json index ef3cec421..6bde09822 100644 --- a/biome.json +++ b/biome.json @@ -34,6 +34,7 @@ "!**/test-sessions.ts", "!**/models.generated.ts", "!**/*.models.ts", + "!packages/evals/src/pi-harness.ts", "!packages/mom/data/**/*", "!!**/node_modules" ] diff --git a/package-lock.json b/package-lock.json index 64003473b..2360d1c06 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5622,6 +5622,7 @@ "name": "@earendil-works/pi-evals", "version": "0.82.1", "devDependencies": { + "@earendil-works/pi-ai": "^0.82.1", "@earendil-works/pi-coding-agent": "^0.82.1", "@types/node": "24.12.4", "shx": "0.4.0", diff --git a/packages/evals/README.md b/packages/evals/README.md index f4b005243..52f29e16a 100644 --- a/packages/evals/README.md +++ b/packages/evals/README.md @@ -2,8 +2,13 @@ Behavioral evaluations for Pi using `vitest-evals`. -The Pi harness in `src/pi-harness.ts` is built on `createHarness` from `vitest-evals/harness`. It drives a real -`createAgentSession` in a temporary workspace and reports the final assistant message, transcript, and token usage. +`src/pi-harness.ts` mirrors the upstream `@vitest-evals/harness-pi-ai` implementation, except that it imports Pi AI +from `@earendil-works/pi-ai/compat`. Keep it synchronized with +[`getsentry/vitest-evals`](https://github.com/getsentry/vitest-evals/blob/main/packages/harness-pi-ai/src/index.ts). + +Pi's `AgentSession` is the system under test. `src/pi-agent.ts` adapts it to the harness's agent contract, while +`piAiHarness(...)` handles eval execution and trace normalization. Each run uses a temporary workspace that is removed +afterward. ## Running diff --git a/packages/evals/package.json b/packages/evals/package.json index 9b466d8c7..20b7ab5ef 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -9,6 +9,7 @@ }, "devDependencies": { "@earendil-works/pi-coding-agent": "^0.82.1", + "@earendil-works/pi-ai": "^0.82.1", "@types/node": "24.12.4", "shx": "0.4.0", "typescript": "5.9.3", diff --git a/packages/evals/src/extensions.eval.ts b/packages/evals/src/extensions.eval.ts new file mode 100644 index 000000000..d09d99695 --- /dev/null +++ b/packages/evals/src/extensions.eval.ts @@ -0,0 +1,49 @@ +import { stat } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; +import { describeEval } from "vitest-evals"; +import { createPiEvalAgent } from "./pi-agent.ts"; +import { piAiHarness } from "./pi-harness.ts"; + +const createHelloExtensionPrompt = + "Create a Pi extension with a hello tool that takes a name and returns a greeting. For example, passing Bob should return `Hello, Bob!`."; + +const piExtensionsHarness = piAiHarness({ + name: "pi-coding-agent-extensions", + agent: ({ context }) => + createPiEvalAgent( + { + run: async ({ input, cwd, prompt, session }) => { + expect(session.extensionRunner.getExtensionPaths()).toHaveLength(0); + await prompt(input); + const extensionPath = join(cwd, ".pi", "extensions", "hello.ts"); + const extensionCreated = await stat(extensionPath).then( + (file) => file.isFile(), + () => false, + ); + if (!extensionCreated) throw new Error(`Extension was not created at ${extensionPath}`); + + await session.reload(); + expect(session.extensionRunner.getExtensionPaths()).toHaveLength(1); + if (!session.getActiveToolNames().includes("hello")) { + throw new Error("Extension tool 'hello' was not loaded"); + } + if (session.getToolDefinition("hello")?.label !== "Hello") { + throw new Error("Extension tool 'hello' did not have label 'Hello'"); + } + + return prompt("Use the hello tool to greet Bob. Respond with exactly the text returned by the tool."); + }, + }, + context.signal, + ), +}); + +describeEval("Pi extensions", { harness: piExtensionsHarness }, (it) => { + it("creates, reloads, and uses a hello extension", async ({ run }) => { + const result = await run(createHelloExtensionPrompt); + + expect(result.output.trim()).toBe("Hello, Bob!"); + expect(result.errors).toEqual([]); + }); +}); diff --git a/packages/evals/src/pi-agent.ts b/packages/evals/src/pi-agent.ts new file mode 100644 index 000000000..340b91eda --- /dev/null +++ b/packages/evals/src/pi-agent.ts @@ -0,0 +1,137 @@ +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + createAgentSessionFromServices, + createAgentSessionServices, + ModelRuntime, + SessionManager, + SettingsManager, +} from "@earendil-works/pi-coding-agent"; +import type { PiAiEventSink, PiAiRuntime, PiAiToolset } from "./pi-harness.ts"; + +type EvalSession = Awaited>["session"]; +type EvalRuntime = PiAiRuntime, string>; + +type PiEvalAgentOptions = { + noTools?: "all" | "builtin"; + run?: (args: { + input: string; + cwd: string; + session: EvalSession; + prompt: (input: string) => Promise; + }) => Promise; +}; + +function getRequiredModelSelection(): { provider: string; model: string } { + const provider = process.env.PI_PROVIDER?.trim(); + const model = process.env.PI_MODEL?.trim(); + if (!provider || !model) { + throw new Error("PI_PROVIDER and PI_MODEL must both be set for eval runs."); + } + return { provider, model }; +} + +async function promptPi( + session: EvalSession, + input: string, + signal: AbortSignal | undefined, + events: PiAiEventSink, + recordUser: boolean, +): Promise { + signal?.throwIfAborted(); + if (recordUser) events.user(input); + await session.prompt(input); + + const assistant = session.messages + .slice() + .reverse() + .find((message) => message.role === "assistant"); + if (!assistant) throw new Error("Pi eval did not produce an assistant message."); + if (assistant.stopReason !== "stop") { + throw new Error(assistant.errorMessage ?? `Pi eval stopped unexpectedly: ${assistant.stopReason}`); + } + + const output = session.getLastAssistantText(); + if (!output) throw new Error("Pi eval produced an empty assistant response."); + events.assistant(output); + return output; +} + +export async function createPiEvalAgent(options: PiEvalAgentOptions, signal: AbortSignal | undefined) { + signal?.throwIfAborted(); + const selection = getRequiredModelSelection(); + const modelRuntime = await ModelRuntime.create(); + const model = modelRuntime.getModel(selection.provider, selection.model); + if (!model) throw new Error(`Eval model not found: ${selection.provider}/${selection.model}`); + + const root = await mkdtemp(join(tmpdir(), "pi-eval-")); + const cwd = join(root, "workspace"); + const agentDir = join(root, "agent"); + let session: EvalSession | undefined; + const cleanup = async () => { + try { + session?.dispose(); + } finally { + await rm(root, { recursive: true, force: true }); + } + }; + + try { + await Promise.all([mkdir(cwd), mkdir(agentDir)]); + const services = await createAgentSessionServices({ + cwd, + agentDir, + modelRuntime, + settingsManager: SettingsManager.inMemory(), + }); + signal?.throwIfAborted(); + session = ( + await createAgentSessionFromServices({ + services, + sessionManager: SessionManager.inMemory(cwd), + model, + thinkingLevel: "off", + noTools: options.noTools, + }) + ).session; + } catch (error) { + await cleanup(); + throw error; + } + + const evalSession = session; + return { + agent: evalSession.agent, + async run(input: string, runtime: EvalRuntime) { + const abort = () => void evalSession.abort(); + let promptCount = 0; + runtime.signal?.addEventListener("abort", abort, { once: true }); + try { + const prompt = (promptInput: string) => { + const recordUser = promptCount > 0; + promptCount++; + return promptPi(evalSession, promptInput, runtime.signal, runtime.events, recordUser); + }; + const output = options.run + ? await options.run({ input, cwd, session: evalSession, prompt }) + : await prompt(input); + const stats = evalSession.getSessionStats(); + return { + output, + metrics: { + provider: model.provider, + model: model.id, + inputTokens: stats.tokens.input, + outputTokens: stats.tokens.output, + totalTokens: stats.tokens.total, + toolCalls: stats.toolCalls, + }, + }; + } finally { + runtime.signal?.removeEventListener("abort", abort); + await cleanup(); + } + }, + }; +} diff --git a/packages/evals/src/pi-harness.ts b/packages/evals/src/pi-harness.ts index dd09ee1e4..0e8c885e2 100644 --- a/packages/evals/src/pi-harness.ts +++ b/packages/evals/src/pi-harness.ts @@ -1,166 +1,1938 @@ -import { mkdir, mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; +import type { + Harness, + HarnessContext, + HarnessRun, + JsonValue, + NormalizedError, + NormalizedSpan, + NormalizedSession, + TranscriptEvent, + TranscriptMessageEvent, + TranscriptToolCallEvent, + NormalizedTrace, + TimingSummary, + UsageSummary, +} from "vitest-evals/harness"; import { - createAgentSession, - DefaultResourceLoader, - ModelRuntime, - SessionManager, - SettingsManager, -} from "@earendil-works/pi-coding-agent"; -import { createHarness, type SimpleHarnessResult } from "vitest-evals/harness"; + attachHarnessRunToError, + createFailedHarnessRun, + createGenAiUsageAttributes, + ensureRunTrace, + getHarnessRunFromError, + isHarnessRun, + isNormalizedSession, + normalizeMetadata, + normalizeContent, + normalizeSpanAttributes, + normalizeSpanError, + resolveHarnessRunErrors, + serializeError, + toJsonValue, +} from "vitest-evals/harness"; +import { + executeWithReplay, + getReplayMetadataFromError, + normalizeReplayMetadata, +} from "vitest-evals/replay"; +import type { + ReplayMode, + ToolRecording, + ToolReplayConfig, +} from "vitest-evals/replay"; +import { + createJudgeHarness, + type JudgeHarnessInput, +} from "vitest-evals/judges"; +import { completeSimple } from "@earendil-works/pi-ai/compat"; +import type { + Api, + AssistantMessage, + Model, + SimpleStreamOptions, +} from "@earendil-works/pi-ai/compat"; -type EvalSession = Awaited>["session"]; +type MaybePromise = T | Promise; +type JsonOutput = [TValue] extends [JsonValue | undefined] + ? TValue + : undefined; +type ResultFieldOutput< + TResult, + TKey extends string, +> = TKey extends keyof TResult + ? Record extends Pick + ? JsonOutput | undefined + : JsonOutput + : undefined; +type AgentSource = + | TAgent + | ((args: PiAiCreateAgentArgs) => MaybePromise); +type AnyPiAiToolset = Record< + string, + PiAiToolDefinition +>; -async function cleanupEvalResources(session: EvalSession | undefined, removeRoot: () => Promise): Promise { - const failures: unknown[] = []; - if (session) { - try { - session.dispose(); - } catch (error) { - failures.push(error); - } - } - try { - await removeRoot(); - } catch (error) { - failures.push(error); - } - if (failures.length === 1) { - throw failures[0]; - } - if (failures.length > 1) { - throw new AggregateError(failures, "Pi eval session disposal and temporary directory cleanup failed."); - } +let nextTraceId = 0; + +/** + * Configuration for adapting a Pi AI model into a judge harness. + * + * @example + * ```ts + * import { getModel } from "@earendil-works/pi-ai/compat"; + * import { piAiJudgeHarness } from "@vitest-evals/harness-pi-ai"; + * + * const judgeHarness = piAiJudgeHarness({ + * model: getModel("anthropic", "claude-sonnet-4-5"), + * temperature: 0, + * }); + * ``` + */ +export interface PiAiJudgeHarnessConfig { + /** Pi AI model used for judge prompts. */ + model: Model; + /** Optional display name for diagnostics. */ + name?: string; + /** Optional judge-model temperature. */ + temperature?: number; + /** Optional judge-model token cap. */ + maxOutputTokens?: number; + /** Additional Pi AI stream options forwarded to `completeSimple(...)`. */ + options?: Omit; } -function getRequiredModelSelection(): { provider: string; model: string } { - const provider = process.env.PI_PROVIDER?.trim(); - const model = process.env.PI_MODEL?.trim(); - if (!provider || !model) { - throw new Error("PI_PROVIDER and PI_MODEL must both be set for eval runs."); - } - return { provider, model }; +/** + * Adapts a Pi AI model into the provider-neutral judge harness API. + * + * @example + * ```ts + * import { getModel } from "@earendil-works/pi-ai/compat"; + * import { piAiJudgeHarness } from "@vitest-evals/harness-pi-ai"; + * import { describeEval, FactualityJudge } from "vitest-evals"; + * + * const judgeHarness = piAiJudgeHarness({ + * model: getModel("anthropic", "claude-sonnet-4-5"), + * temperature: 0, + * }); + * + * describeEval("qa agent", { + * harness: qaHarness, + * judgeHarness, + * judges: [FactualityJudge()], + * }, (it) => {}); + * ``` + */ +export function piAiJudgeHarness( + config: PiAiJudgeHarnessConfig, +) { + return createJudgeHarness({ + name: config.name ?? "pi-ai", + run: async ({ responseFormat, system, prompt }, { signal }) => { + const message = await completeSimple( + config.model, + { + systemPrompt: formatPiAiJudgeSystemPrompt(system, responseFormat), + messages: [ + { + role: "user", + content: prompt, + timestamp: Date.now(), + }, + ], + }, + { + ...(config.options ?? {}), + ...(config.temperature !== undefined + ? { temperature: config.temperature } + : {}), + ...(config.maxOutputTokens !== undefined + ? { maxTokens: config.maxOutputTokens } + : {}), + ...(signal ? { signal } : {}), + }, + ); + + return resolvePiAiJudgeText(message); + }, + }); } -async function runPiEval(input: string, signal: AbortSignal | undefined): Promise> { - signal?.throwIfAborted(); - const selection = getRequiredModelSelection(); - const modelRuntime = await ModelRuntime.create(); - signal?.throwIfAborted(); - const model = modelRuntime.getModel(selection.provider, selection.model); - if (!model) { - throw new Error(`Eval model not found: ${selection.provider}/${selection.model}`); - } - const auth = await modelRuntime.checkAuth(selection.provider); - signal?.throwIfAborted(); - if (!auth) { - throw new Error(`No authentication configured for eval model ${selection.provider}/${selection.model}.`); - } +function formatPiAiJudgeSystemPrompt( + system: string | undefined, + responseFormat: JudgeHarnessInput["responseFormat"], +) { + if (responseFormat?.type !== "json") { + return system; + } - const root = await mkdtemp(join(tmpdir(), "pi-eval-")); - const cwd = join(root, "workspace"); - const agentDir = join(root, "agent"); - const cleanup = () => rm(root, { recursive: true, force: true }); - let createdSession: EvalSession | undefined; - try { - signal?.throwIfAborted(); - await Promise.all([mkdir(cwd), mkdir(agentDir)]); - signal?.throwIfAborted(); + const jsonInstruction = [ + "Return only valid JSON. Do not include markdown fences or explanatory prose.", + responseFormat.schema + ? `JSON Schema:\n${JSON.stringify(responseFormat.schema, null, 2)}` + : undefined, + ] + .filter(Boolean) + .join("\n\n"); - const settingsManager = SettingsManager.inMemory(); - const resourceLoader = new DefaultResourceLoader({ - cwd, - agentDir, - settingsManager, - noExtensions: true, - noSkills: true, - noPromptTemplates: true, - noThemes: true, - noContextFiles: true, - }); - await resourceLoader.reload(); - signal?.throwIfAborted(); - - createdSession = ( - await createAgentSession({ - cwd, - agentDir, - model, - modelRuntime, - thinkingLevel: "off", - noTools: "all", - resourceLoader, - settingsManager, - sessionManager: SessionManager.inMemory(cwd), - }) - ).session; - signal?.throwIfAborted(); - } catch (error) { - try { - await cleanupEvalResources(createdSession, cleanup); - } catch (cleanupError) { - throw new AggregateError([error, cleanupError], "Pi eval setup failed and cleanup also failed."); - } - throw error; - } - if (!createdSession) { - throw new Error("Pi eval session setup completed without a session."); - } - - const session = createdSession; - const abort = () => void session.abort(); - signal?.addEventListener("abort", abort, { once: true }); - let result: SimpleHarnessResult; - try { - signal?.throwIfAborted(); - await session.prompt(input); - const assistant = session.messages - .slice() - .reverse() - .find((message) => message.role === "assistant"); - if (!assistant) { - throw new Error("Pi eval did not produce an assistant message."); - } - if (assistant.stopReason !== "stop") { - throw new Error(assistant.errorMessage ?? `Pi eval stopped unexpectedly: ${assistant.stopReason}`); - } - - const output = session.getLastAssistantText(); - if (!output) { - throw new Error("Pi eval produced an empty assistant response."); - } - - const stats = session.getSessionStats(); - result = { - output, - events: [ - { type: "message", role: "user", content: input }, - { type: "message", role: "assistant", content: output }, - ], - usage: { - provider: model.provider, - model: model.id, - inputTokens: stats.tokens.input, - outputTokens: stats.tokens.output, - totalTokens: stats.tokens.total, - }, - }; - } catch (error) { - signal?.removeEventListener("abort", abort); - try { - await cleanupEvalResources(session, cleanup); - } catch (cleanupError) { - throw new AggregateError([error, cleanupError], "Pi eval run failed and cleanup also failed."); - } - throw error; - } - signal?.removeEventListener("abort", abort); - await cleanupEvalResources(session, cleanup); - return result; + return system ? `${system}\n\n${jsonInstruction}` : jsonInstruction; } -export const piCodingAgentHarness = createHarness({ - name: "pi-coding-agent", - run: ({ input, signal }) => runPiEval(input, signal), -}); +function resolvePiAiJudgeText(message: AssistantMessage) { + if (message.stopReason === "error" || message.stopReason === "aborted") { + throw new Error(message.errorMessage ?? "Pi AI judge harness failed."); + } + + return message.content + .filter((part) => part.type === "text") + .map((part) => part.text) + .join("") + .trim(); +} + +type InferredPiAiToolset = Record< + string, + PiAiToolDefinition, JsonValue, TInput> +>; + +type PiAgentToolLike = { + name: string; + execute: (toolCallId: string, args: Record) => unknown; +}; + +type PiAiResultOutput = TResult extends HarnessRun + ? TOutput + : "output" extends keyof TResult + ? ResultFieldOutput + : undefined; +type PiAiAgentResult< + TAgent, + TInput, + TTools extends PiAiToolset, +> = TAgent extends { + run: ( + input: TInput, + runtime: PiAiRuntime, + ) => MaybePromise; +} + ? Awaited + : unknown; + +const ORIGINAL_NATIVE_EXECUTE = Symbol("vitest-evals.originalNativeExecute"); + +type NativeToolExecute = PiAgentToolLike["execute"] & { + [ORIGINAL_NATIVE_EXECUTE]?: PiAgentToolLike["execute"]; +}; + +/** Replay mode alias used by the Pi AI harness package. */ +export type PiAiReplayMode = ReplayMode; + +/** Event sink for recording Pi AI transcript events. */ +export interface PiAiEventSink { + message: (event: TranscriptMessageEvent) => void; + system: (content: JsonValue, metadata?: Record) => void; + user: (content: JsonValue, metadata?: Record) => void; + assistant: (content: JsonValue, metadata?: Record) => void; + /** Records a tool-result event linked to an assistant tool call by id. */ + tool: ( + toolCallId: string, + content: JsonValue, + options?: { + name?: string; + error?: NormalizedError; + metadata?: Record; + }, + ) => void; +} + +/** Context passed to instrumented Pi AI tool executions. */ +export interface PiAiToolContext { + input: TInput; + signal?: AbortSignal; + setArtifact: HarnessContext["setArtifact"]; +} + +/** Tool replay recording shape for Pi AI tools. */ +export type PiAiToolRecording< + TArgs extends Record = Record, + TResult extends JsonValue = JsonValue, +> = ToolRecording; + +/** Replay configuration for a Pi AI tool. */ +export type PiAiToolReplayConfig< + TArgs extends Record = Record, + TResult extends JsonValue = JsonValue, + TInput = string, +> = ToolReplayConfig>; + +/** Replay policy for one Pi AI tool. */ +export type PiAiToolReplayPolicy = + | boolean + | PiAiToolReplayConfig, JsonValue, TInput>; + +/** Replay policy map keyed by Pi AI tool name. */ +export type PiAiToolReplayPolicies = Record< + string, + PiAiToolReplayPolicy +>; + +/** Tool definition accepted by the Pi AI harness runtime. */ +export interface PiAiToolDefinition< + TArgs extends Record = Record, + TResult extends JsonValue = JsonValue, + TInput = string, +> { + description?: string; + execute: ( + args: TArgs, + context: PiAiToolContext, + ) => MaybePromise; +} + +/** Toolset shape accepted by the Pi AI harness. */ +export type PiAiToolset = AnyPiAiToolset; + +type ToolArgs = TTool extends PiAiToolDefinition< + infer TArgs, + infer _TResult, + infer _TInput +> + ? TArgs + : never; + +type ToolResult = TTool extends PiAiToolDefinition< + infer _TArgs, + infer TResult, + infer _TInput +> + ? TResult + : never; + +/** Runtime object passed into Pi AI agent entrypoints. */ +export type PiAiRuntime, TInput = string> = { + tools: { + [K in keyof TTools]: ( + args: ToolArgs, + ) => Promise>; + }; + events: PiAiEventSink; + signal?: AbortSignal; +}; + +/** Arguments passed to custom Pi AI harness run callbacks. */ +export interface PiAiHarnessRunArgs< + TAgent, + TInput, + TTools extends PiAiToolset, +> { + agent: TAgent; + input: TInput; + context: HarnessContext; + runtime: PiAiRuntime; +} + +/** Arguments passed to Pi AI harness output selectors. */ +export interface PiAiHarnessResultArgs< + TAgent, + TInput, + TResult, + TTools extends PiAiToolset, +> extends PiAiHarnessRunArgs { + result: TResult; +} + +/** Arguments passed to per-run Pi AI agent factories. */ +export interface PiAiCreateAgentArgs { + input: TInput; + context: HarnessContext; +} + +interface PiAiHarnessBaseOptions< + TAgent, + TInput = string, + TResult = unknown, + TTools extends PiAiToolset = PiAiToolset, + TOutput extends JsonValue | undefined = JsonValue | undefined, +> { + agent: AgentSource; + toolReplay?: PiAiToolReplayPolicies; + output?: PiAiHarnessOutputSelector; + name?: string; +} + +interface PiAiHarnessWithToolsOptions< + TAgent, + TInput = string, + TResult = unknown, + TTools extends PiAiToolset = PiAiToolset, + TOutput extends JsonValue | undefined = JsonValue | undefined, +> extends PiAiHarnessBaseOptions { + tools: TTools; + run?: ( + args: PiAiHarnessRunArgs, + ) => MaybePromise>; +} + +interface PiAiHarnessInferredToolsOptions< + TAgent, + TInput = string, + TResult = unknown, + TOutput extends JsonValue | undefined = JsonValue | undefined, +> extends PiAiHarnessBaseOptions< + TAgent, + TInput, + TResult, + InferredPiAiToolset, + TOutput + > { + tools?: undefined; + run?: ( + args: PiAiHarnessRunArgs>, + ) => MaybePromise>; +} + +type PiAiHarnessOptions< + TAgent, + TInput = string, + TResult = unknown, + TTools extends PiAiToolset = PiAiToolset, + TOutput extends JsonValue | undefined = JsonValue | undefined, +> = + | PiAiHarnessWithToolsOptions + | PiAiHarnessInferredToolsOptions; + +type PiAiHarnessWithToolsOptionsWithOutput< + TAgent, + TInput = string, + TResult = unknown, + TTools extends PiAiToolset = PiAiToolset, + TOutput extends JsonValue | undefined = JsonValue | undefined, +> = PiAiHarnessWithToolsOptions & { + output: PiAiHarnessOutputSelector; +}; + +type PiAiHarnessInferredToolsOptionsWithOutput< + TAgent, + TInput = string, + TResult = unknown, + TOutput extends JsonValue | undefined = JsonValue | undefined, +> = PiAiHarnessInferredToolsOptions & { + output: PiAiHarnessOutputSelector< + TAgent, + TInput, + TResult, + InferredPiAiToolset, + TOutput + >; +}; + +type PiAiHarnessWithToolsRunOptionsWithoutOutput< + TAgent, + TInput = string, + TResult = unknown, + TTools extends PiAiToolset = PiAiToolset, +> = PiAiHarnessBaseOptions< + TAgent, + TInput, + TResult, + TTools, + JsonValue | undefined +> & { + tools: TTools; + run: ( + args: PiAiHarnessRunArgs, + ) => MaybePromise; + output?: never; +}; + +type PiAiHarnessWithToolsAgentOptionsWithoutOutput< + TAgent, + TInput = string, + TTools extends PiAiToolset = PiAiToolset, +> = PiAiHarnessBaseOptions< + TAgent, + TInput, + unknown, + TTools, + JsonValue | undefined +> & { + tools: TTools; + run?: never; + output?: never; +}; + +type PiAiHarnessInferredToolsRunOptionsWithoutOutput< + TAgent, + TInput = string, + TResult = unknown, +> = PiAiHarnessBaseOptions< + TAgent, + TInput, + TResult, + InferredPiAiToolset, + JsonValue | undefined +> & { + tools?: undefined; + run: ( + args: PiAiHarnessRunArgs>, + ) => MaybePromise; + output?: never; +}; + +type PiAiHarnessInferredToolsAgentOptionsWithoutOutput< + TAgent, + TInput = string, +> = PiAiHarnessBaseOptions< + TAgent, + TInput, + unknown, + InferredPiAiToolset, + JsonValue | undefined +> & { + tools?: undefined; + run?: never; + output?: never; +}; + +type PiAiHarnessRunOptions< + TAgent, + TInput, + TResult, + TTools extends PiAiToolset, + TOutput extends JsonValue | undefined, +> = PiAiHarnessBaseOptions & { + run?: ( + args: PiAiHarnessRunArgs, + ) => MaybePromise>; +}; + +type PiAiRunnableAgent< + TInput, + TResult, + TOutput extends JsonValue | undefined, + TTools extends PiAiToolset, +> = { + run: ( + input: TInput, + runtime: PiAiRuntime, + ) => MaybePromise>; +}; + +type PiAiHarnessOutputSelector< + TAgent, + TInput = string, + TResult = unknown, + TTools extends PiAiToolset = PiAiToolset, + TOutput extends JsonValue | undefined = JsonValue | undefined, +> = ( + args: PiAiHarnessResultArgs, +) => MaybePromise; + +type InferredToolSurfaces = { + runtimeTools?: InferredPiAiToolset; + nativeToolsets?: Array[]>; +}; + +type PiToolExecutionState = { + activeNativeToolNames: Map; +}; + +/** Adapts a Pi agent runtime into a normalized vitest-evals harness. */ +export function piAiHarness< + TAgent, + TInput = string, + TResult = unknown, + TTools extends PiAiToolset = PiAiToolset, + TOutput extends JsonValue | undefined = JsonValue | undefined, +>( + options: PiAiHarnessWithToolsOptionsWithOutput< + TAgent, + TInput, + TResult, + TTools, + TOutput + >, +): Harness; +export function piAiHarness< + TAgent, + TInput = string, + TResult = unknown, + TOutput extends JsonValue | undefined = JsonValue | undefined, +>( + options: PiAiHarnessInferredToolsOptionsWithOutput< + TAgent, + TInput, + TResult, + TOutput + >, +): Harness; +export function piAiHarness< + TAgent, + TInput = string, + TResult = unknown, + TTools extends PiAiToolset = PiAiToolset, +>( + options: PiAiHarnessWithToolsRunOptionsWithoutOutput< + TAgent, + TInput, + TResult, + TTools + >, +): Harness>>; +export function piAiHarness< + TAgent, + TInput = string, + TTools extends PiAiToolset = PiAiToolset, +>( + options: PiAiHarnessWithToolsAgentOptionsWithoutOutput< + TAgent, + TInput, + TTools + >, +): Harness>>; +export function piAiHarness( + options: PiAiHarnessInferredToolsRunOptionsWithoutOutput< + TAgent, + TInput, + TResult + >, +): Harness>>; +export function piAiHarness( + options: PiAiHarnessInferredToolsAgentOptionsWithoutOutput, +): Harness< + TInput, + PiAiResultOutput>> +>; +export function piAiHarness< + TAgent, + TInput = string, + TResult = unknown, + TTools extends PiAiToolset = PiAiToolset, + TOutput extends JsonValue | undefined = JsonValue | undefined, +>( + options: PiAiHarnessOptions, +): Harness { + validateOptions(options); + + const harnessName = options.name ?? "pi-ai"; + const harness: Harness = { + name: harnessName, + run: async (input, context) => { + const startedAt = new Date(); + + try { + const agent = await resolveAgent(options, { + input, + context, + }); + const events: TranscriptEvent[] = [ + { + type: "message", + role: "user", + content: normalizeContent(input), + }, + ]; + const inferredTools = resolveInferredToolSurfaces(agent); + + if (hasExplicitToolset(options)) { + return await executePiHarnessRun( + options, + agent, + input, + context, + events, + options.tools, + inferredTools.nativeToolsets, + ); + } + + return await executePiHarnessRun( + options, + agent, + input, + context, + events, + inferredTools.runtimeTools, + inferredTools.nativeToolsets, + ); + } catch (error) { + if (getHarnessRunFromError(error)) { + throw error; + } + + throw attachHarnessRunToError( + error, + createFailedPiAiRun(input, context, error, harnessName, startedAt), + ); + } + }, + }; + + return harness; +} + +function createFailedPiAiRun( + input: TInput, + context: HarnessContext, + error: unknown, + harnessName: string, + startedAt: Date, +) { + const run = createFailedHarnessRun(input, error, { + artifacts: context.artifacts, + }); + ensureRunTrace(run, { + name: harnessName, + startedAt, + finishedAt: new Date(), + operationName: "invoke_agent", + source: "harness-pi-ai", + }); + + return run; +} + +function validateOptions< + TAgent, + TInput, + TResult, + TTools extends PiAiToolset, + TOutput extends JsonValue | undefined, +>(options: PiAiHarnessOptions) { + if (options.agent === undefined) { + throw new Error( + "piAiHarness requires agent. Pass an agent instance or an agent factory.", + ); + } +} + +async function executePiHarnessRun< + TAgent, + TInput, + TResult, + TTools extends PiAiToolset, + TOutput extends JsonValue | undefined, +>( + options: PiAiHarnessRunOptions, + agent: TAgent, + input: TInput, + context: HarnessContext, + events: TranscriptEvent[], + runtimeTools: TTools | undefined, + nativeToolsets?: Array[]>, +): Promise> { + const trace = createTraceRecorder(options.name ?? "pi-ai"); + const executionState = createPiToolExecutionState(); + const runtime = createRuntime({ + input, + context, + tools: runtimeTools, + toolReplay: options.toolReplay, + executionState, + events, + }); + + try { + const result = await withInstrumentedAgentTools( + agent, + nativeToolsets, + { + input, + context, + events, + toolReplay: options.toolReplay, + executionState, + }, + () => + runAgent(options, { + agent, + input, + context, + runtime, + }), + ); + + if (isHarnessRun(result) && !hasOutputSelector(options)) { + if (Object.keys(context.artifacts).length > 0 && !result.artifacts) { + result.artifacts = context.artifacts; + } + attachPiAiTrace(result, trace, new Date()); + return result; + } + + const normalizeResult = result as TResult; + const resultArgs = { + agent, + input, + context, + runtime, + result: normalizeResult, + } satisfies PiAiHarnessResultArgs; + + const output = options.output + ? await options.output(resultArgs) + : (resolveOutput(normalizeResult) as TOutput | undefined); + const usage = resolveUsage(normalizeResult, countToolCallEvents(events)); + const session = resolveSession(normalizeResult, events, output, usage); + const errors = resolveErrors(normalizeResult); + const finishedAt = new Date(); + + return { + session, + output, + usage, + artifacts: + Object.keys(context.artifacts).length > 0 + ? context.artifacts + : undefined, + errors, + traces: [ + finishPiAiTrace(trace, { + session, + usage, + errors, + finishedAt, + }), + ], + } as HarnessRun; + } catch (error) { + const usage = resolveUsage(undefined, countToolCallEvents(events)); + const session = resolveSession(undefined, events, undefined, usage); + const finishedAt = new Date(); + const serializedError = serializeError(error); + const run = { + session, + output: undefined, + usage, + artifacts: + Object.keys(context.artifacts).length > 0 + ? context.artifacts + : undefined, + errors: [serializedError], + traces: [ + finishPiAiTrace(trace, { + session, + usage, + errors: [serializedError], + finishedAt, + }), + ], + } satisfies HarnessRun; + + throw attachHarnessRunToError(error, run); + } +} + +async function resolveAgent< + TAgent, + TInput, + TResult, + TTools extends PiAiToolset, + TOutput extends JsonValue | undefined, +>( + options: PiAiHarnessOptions, + args: PiAiCreateAgentArgs, +) { + return resolveAgentSource(options.agent, args); +} + +async function resolveAgentSource( + agent: AgentSource, + args: PiAiCreateAgentArgs, +): Promise { + if (isAgentFactory(agent)) { + return agent(args); + } + + return agent; +} + +function isAgentFactory( + agent: AgentSource, +): agent is (args: PiAiCreateAgentArgs) => MaybePromise { + return typeof agent === "function" && !hasPiAiRunMethod(agent); +} + +function hasOutputSelector< + TAgent, + TInput, + TResult, + TTools extends PiAiToolset, + TOutput extends JsonValue | undefined, +>(options: PiAiHarnessRunOptions) { + return Boolean(options.output); +} + +type TraceRecorder = { + id: string; + rootSpanId: string; + name: string; + startedAt: Date; +}; + +function createTraceRecorder(name: string): TraceRecorder { + const id = `pi_ai_trace_${++nextTraceId}`; + + return { + id, + rootSpanId: `${id}:run`, + name, + startedAt: new Date(), + }; +} + +function attachPiAiTrace( + run: HarnessRun, + trace: TraceRecorder, + finishedAt: Date, +) { + const nativeTrace = finishPiAiTrace(trace, { + session: run.session, + usage: run.usage, + errors: run.errors ?? [], + finishedAt, + }); + + run.traces = [...(run.traces ?? []), nativeTrace]; +} + +function finishPiAiTrace( + trace: TraceRecorder, + options: { + session: NormalizedSession; + usage?: UsageSummary; + errors?: unknown[]; + finishedAt: Date; + }, +): NormalizedTrace { + const finishedAt = options.finishedAt; + const durationMs = finishedAt.getTime() - trace.startedAt.getTime(); + const rootError = options.errors?.[0] + ? normalizeSpanError(options.errors[0]) + : undefined; + const rootSpan: NormalizedSpan = { + id: trace.rootSpanId, + traceId: trace.id, + name: trace.name, + kind: "run", + startedAt: trace.startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + durationMs, + status: rootError ? "error" : "ok", + ...(rootError ? { error: rootError } : {}), + attributes: normalizeSpanAttributes({ + "gen_ai.operation.name": "invoke_agent", + "gen_ai.workflow.name": trace.name, + ...createGenAiUsageAttributes(options.usage), + }), + }; + const modelSpan = createUsageModelSpan(trace, options.usage); + const spans = [rootSpan, ...(modelSpan ? [modelSpan] : [])]; + + return { + id: trace.id, + name: trace.name, + startedAt: trace.startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + durationMs, + metadata: { + source: "harness-pi-ai", + }, + spans, + }; +} + +function createUsageModelSpan( + trace: TraceRecorder, + usage: UsageSummary | undefined, +): NormalizedSpan | undefined { + if (!usage?.provider && !usage?.model) { + return undefined; + } + + return { + id: `${trace.id}:model:1`, + traceId: trace.id, + parentId: trace.rootSpanId, + name: usage.model ? `pi-ai ${usage.model}` : "pi-ai model", + kind: "model", + status: "ok", + attributes: normalizeSpanAttributes({ + "gen_ai.operation.name": "chat", + ...createGenAiUsageAttributes(usage), + }), + }; +} + +function resolveInferredToolSurfaces( + agent: unknown, +): InferredToolSurfaces { + let runtimeTools: InferredPiAiToolset | undefined; + const nativeToolsets: Array[]> = []; + const seenToolsets = new Set[]>(); + + for (const candidate of getAgentToolCandidates(agent)) { + const nextRuntimeTools = getRuntimeToolset(candidate); + if (runtimeTools === undefined && nextRuntimeTools !== undefined) { + runtimeTools = nextRuntimeTools; + } + + const nativeTools = getNativeToolArray(candidate); + if (nativeTools && !seenToolsets.has(nativeTools)) { + seenToolsets.add(nativeTools); + nativeToolsets.push(nativeTools); + } + } + + return { + ...(runtimeTools ? { runtimeTools } : {}), + ...(nativeToolsets.length > 0 ? { nativeToolsets } : {}), + }; +} + +function getAgentToolCandidates(agent: unknown): object[] { + const roots = getAgentRoots(agent); + const candidates: object[] = []; + const seen = new Set(); + + for (const root of roots) { + addUniqueObject(candidates, seen, root); + addUniqueObject(candidates, seen, getObjectProperty(root, "state")); + addUniqueObject(candidates, seen, getObjectProperty(root, "initialState")); + } + + return candidates; +} + +function getAgentRoots(agent: unknown): object[] { + return [asObject(agent)] + .concat(asObject(getObjectProperty(agent, "agent"))) + .filter((value): value is object => value !== undefined); +} + +function addUniqueObject( + candidates: object[], + seen: Set, + value: unknown, +) { + if (!value || typeof value !== "object" || seen.has(value)) { + return; + } + + seen.add(value); + candidates.push(value); +} + +function asObject(value: unknown): object | undefined { + return value && typeof value === "object" ? value : undefined; +} + +function getObjectProperty(value: unknown, key: string): unknown { + return value && typeof value === "object" + ? (value as Record)[key] + : undefined; +} + +function getRuntimeToolset( + value: object, +): InferredPiAiToolset | undefined { + const candidate = + getObjectProperty(value, "tools") ?? getObjectProperty(value, "toolset"); + + return isPiAiToolset(candidate) + ? (candidate as InferredPiAiToolset) + : undefined; +} + +function getNativeToolArray( + value: object, +): PiAgentToolLike[] | undefined { + const candidate = getObjectProperty(value, "tools"); + if (isAgentToolArray(candidate)) { + return candidate as PiAgentToolLike[]; + } + + return undefined; +} + +async function runAgent< + TAgent, + TInput, + TResult, + TTools extends PiAiToolset, + TOutput extends JsonValue | undefined, +>( + options: PiAiHarnessRunOptions, + args: PiAiHarnessRunArgs, +): Promise> { + if (options.run) { + return options.run(args); + } + + if (hasPiAiRunMethod(args.agent)) { + return args.agent.run(args.input, args.runtime); + } + + throw new Error( + "piAiHarness requires a run() function unless the provided agent exposes run(input, runtime).", + ); +} + +function hasExplicitToolset< + TAgent, + TInput, + TResult, + TTools extends PiAiToolset, + TOutput extends JsonValue | undefined, +>( + options: PiAiHarnessOptions, +): options is PiAiHarnessWithToolsOptions< + TAgent, + TInput, + TResult, + TTools, + TOutput +> { + return options.tools !== undefined; +} + +function hasPiAiRunMethod< + TInput, + TResult, + TOutput extends JsonValue | undefined, + TTools extends PiAiToolset, +>( + agent: unknown, +): agent is PiAiRunnableAgent { + if (!agent || (typeof agent !== "object" && typeof agent !== "function")) { + return false; + } + + return ( + "run" in agent && typeof (agent as { run?: unknown }).run === "function" + ); +} + +function isPiAiToolset(value: unknown): value is PiAiToolset { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return false; + } + + const tools = Object.values(value); + return ( + tools.length > 0 && + tools.every((tool) => + Boolean( + tool && + typeof tool === "object" && + "execute" in tool && + typeof (tool as { execute?: unknown }).execute === "function", + ), + ) + ); +} + +function isAgentToolArray(value: unknown): value is PiAgentToolLike[] { + return ( + Array.isArray(value) && + value.every((tool) => + Boolean( + tool && + typeof tool === "object" && + "name" in tool && + typeof (tool as { name?: unknown }).name === "string" && + "execute" in tool && + typeof (tool as { execute?: unknown }).execute === "function", + ), + ) + ); +} + +async function withInstrumentedAgentTools( + agent: unknown, + toolsets: Array[]> | undefined, + args: { + input: TInput; + context: HarnessContext; + events: TranscriptEvent[]; + toolReplay: PiAiToolReplayPolicies | undefined; + executionState: PiToolExecutionState; + }, + callback: () => Promise, +) { + if (!toolsets || toolsets.length === 0) { + return callback(); + } + + const originalExecutions = new Map< + PiAgentToolLike, + PiAgentToolLike["execute"] + >(); + const originalResets = new Map(); + + const patchTool = (tool: PiAgentToolLike) => { + if (originalExecutions.has(tool)) { + return; + } + + const originalExecute = getNativeToolExecuteOrigin(tool.execute); + originalExecutions.set(tool, originalExecute); + const instrumentedExecute: NativeToolExecute = async ( + toolCallId: string, + rawArgs: Record, + ) => { + const startedAt = new Date(); + const toolContext = { + input: args.input, + signal: args.context.signal, + setArtifact: args.context.setArtifact, + } satisfies PiAiToolContext; + const leaveNativeTool = enterNativeToolExecution( + args.executionState, + tool.name, + ); + const call: TranscriptToolCallEvent = { + type: "tool_call", + id: toolCallId, + name: tool.name, + arguments: rawArgs, + startedAt: startedAt.toISOString(), + metadata: normalizeReplayMetadata(undefined), + } satisfies TranscriptToolCallEvent; + args.events.push(call); + + try { + const execution = await executeNativeToolWithReplay({ + toolName: tool.name, + toolCallId, + execute: originalExecute, + replay: args.toolReplay?.[tool.name], + args: rawArgs, + context: toolContext, + }); + const finishedAt = new Date(); + call.finishedAt = finishedAt.toISOString(); + call.durationMs = finishedAt.getTime() - startedAt.getTime(); + call.metadata = normalizeReplayMetadata(execution.replay); + args.events.push({ + type: "tool_result", + toolCallId, + name: tool.name, + content: execution.normalizedResult, + startedAt: call.startedAt, + finishedAt: call.finishedAt, + durationMs: call.durationMs, + }); + return execution.result; + } catch (error) { + const finishedAt = new Date(); + call.finishedAt = finishedAt.toISOString(); + call.durationMs = finishedAt.getTime() - startedAt.getTime(); + call.metadata = normalizeReplayMetadata( + getReplayMetadataFromError(error), + ); + args.events.push({ + type: "tool_result", + toolCallId, + name: tool.name, + error: serializeToolCallError(error), + startedAt: call.startedAt, + finishedAt: call.finishedAt, + durationMs: call.durationMs, + }); + throw error; + } finally { + leaveNativeTool(); + } + }; + instrumentedExecute[ORIGINAL_NATIVE_EXECUTE] = originalExecute; + tool.execute = instrumentedExecute; + }; + + const patchToolsets = (nextToolsets: Array[]>) => { + for (const toolset of nextToolsets) { + for (const tool of toolset) { + patchTool(tool); + } + } + }; + + patchToolsets(toolsets); + + for (const target of getAgentResetTargets(agent)) { + const originalReset = target.reset; + originalResets.set(target, originalReset); + target.reset = function patchedReset(this: ResettableAgent, ...resetArgs) { + const resetResult = originalReset.apply(this, resetArgs); + + if (isPromiseLike(resetResult)) { + return resetResult.finally(() => { + patchToolsets(resolveInferredNativeToolsets(agent)); + }); + } + + patchToolsets(resolveInferredNativeToolsets(agent)); + return resetResult; + }; + } + + try { + return await callback(); + } finally { + for (const [target, originalReset] of originalResets) { + target.reset = originalReset; + } + + for (const [tool, originalExecute] of originalExecutions) { + tool.execute = originalExecute; + } + } +} + +type ResettableAgent = { + reset: (...args: unknown[]) => unknown; +}; + +function getAgentResetTargets(agent: unknown): ResettableAgent[] { + return getAgentRoots(agent).filter(isResettableAgent); +} + +function isResettableAgent(value: object): value is ResettableAgent { + return "reset" in value && typeof value.reset === "function"; +} + +function resolveInferredNativeToolsets( + agent: unknown, +): Array[]> { + const toolsets: Array[]> = []; + const seenToolsets = new Set[]>(); + + for (const candidate of getAgentToolCandidates(agent)) { + const nativeTools = getNativeToolArray(candidate); + if (nativeTools && !seenToolsets.has(nativeTools)) { + seenToolsets.add(nativeTools); + toolsets.push(nativeTools); + } + } + + return toolsets; +} + +function isPromiseLike(value: unknown): value is Promise { + return Boolean( + value && typeof (value as { then?: unknown }).then === "function", + ); +} + +function serializeToolCallError(error: unknown): NormalizedError { + const serialized = serializeError(error); + const { message, type, ...details } = serialized; + + return { + ...details, + message: typeof message === "string" ? message : String(message), + ...(typeof type === "string" ? { type } : {}), + }; +} + +/** Creates a runtime-local id used on both assistant tool calls and tool results. */ +function createRuntimeToolCallId(toolName: string, index: number) { + return `runtime:${toolName}:${index + 1}`; +} + +function getNativeToolExecuteOrigin( + execute: PiAgentToolLike["execute"], +) { + const nativeExecute = execute as NativeToolExecute; + return nativeExecute[ORIGINAL_NATIVE_EXECUTE] ?? nativeExecute; +} + +function createPiToolExecutionState(): PiToolExecutionState { + return { + activeNativeToolNames: new Map(), + }; +} + +function enterNativeToolExecution( + state: PiToolExecutionState, + toolName: string, +) { + state.activeNativeToolNames.set( + toolName, + (state.activeNativeToolNames.get(toolName) ?? 0) + 1, + ); + + return () => { + const nextCount = (state.activeNativeToolNames.get(toolName) ?? 1) - 1; + if (nextCount <= 0) { + state.activeNativeToolNames.delete(toolName); + return; + } + + state.activeNativeToolNames.set(toolName, nextCount); + }; +} + +function hasActiveNativeToolExecution( + state: PiToolExecutionState, + toolName: string, +) { + return (state.activeNativeToolNames.get(toolName) ?? 0) > 0; +} + +async function executeNativeToolWithReplay({ + toolName, + toolCallId, + execute, + replay, + args, + context, +}: { + toolName: string; + toolCallId: string; + execute: PiAgentToolLike["execute"]; + replay: PiAiToolReplayPolicy | undefined; + args: Record; + context: PiAiToolContext; +}) { + let didExecute = false; + let liveResult: unknown; + + const execution = await executeWithReplay({ + toolName: createNativeReplayToolName(toolName), + args, + context, + execute: async (toolArgs) => { + didExecute = true; + liveResult = await execute(toolCallId, toolArgs); + return createNativeToolReplayEnvelope(liveResult); + }, + replay, + }); + + if (didExecute) { + return { + result: liveResult, + normalizedResult: normalizeReplayToolResult(liveResult), + replay: execution.replay, + }; + } + + return { + ...resolveNativeToolReplayResult(execution.result), + replay: execution.replay, + }; +} + +function createNativeReplayToolName(toolName: string) { + return `${toolName}.native`; +} + +function createRuntime>({ + input, + context, + tools, + toolReplay, + executionState, + events, +}: { + input: TInput; + context: HarnessContext; + tools: TTools | undefined; + toolReplay: PiAiToolReplayPolicies | undefined; + executionState: PiToolExecutionState; + events: TranscriptEvent[]; +}): PiAiRuntime { + const eventSink: PiAiEventSink = { + message: (event) => { + events.push(event); + }, + system: (content, metadata) => { + events.push({ + type: "message", + role: "system", + content, + metadata, + }); + }, + user: (content, metadata) => { + events.push({ + type: "message", + role: "user", + content, + metadata, + }); + }, + assistant: (content, metadata) => { + events.push({ + type: "message", + role: "assistant", + content, + metadata, + }); + }, + tool: (toolCallId, content, options) => { + events.push({ + type: "tool_result", + toolCallId, + ...(options?.name ? { name: options.name } : {}), + content, + ...(options?.error ? { error: options.error } : {}), + ...(options?.metadata ? { metadata: options.metadata } : {}), + }); + }, + }; + + const runtimeTools = Object.fromEntries( + Object.entries(tools ?? {}).map(([toolName, tool]) => [ + toolName, + async (args: Record) => { + const startedAt = new Date(); + const isNativeImplementationCall = hasActiveNativeToolExecution( + executionState, + toolName, + ); + const toolContext = { + input, + signal: context.signal, + setArtifact: context.setArtifact, + } satisfies PiAiToolContext; + const toolCallId = createRuntimeToolCallId( + toolName, + countToolCallEvents(events), + ); + const call: TranscriptToolCallEvent = { + type: "tool_call", + id: toolCallId, + name: toolName, + arguments: args, + startedAt: startedAt.toISOString(), + } satisfies TranscriptToolCallEvent; + + try { + if (!isNativeImplementationCall) { + events.push(call); + } + + const execution = await executeToolWithReplay({ + toolName, + tool, + replay: isNativeImplementationCall + ? undefined + : toolReplay?.[toolName], + args, + context: toolContext, + }); + const finishedAt = new Date(); + + if (isNativeImplementationCall) { + return execution.result; + } + + call.finishedAt = finishedAt.toISOString(); + call.durationMs = finishedAt.getTime() - startedAt.getTime(); + call.metadata = normalizeReplayMetadata(execution.replay); + events.push({ + type: "tool_result", + toolCallId, + name: toolName, + content: execution.result, + startedAt: call.startedAt, + finishedAt: call.finishedAt, + durationMs: call.durationMs, + }); + return execution.result; + } catch (error) { + const finishedAt = new Date(); + if (isNativeImplementationCall) { + throw error; + } + + call.finishedAt = finishedAt.toISOString(); + call.durationMs = finishedAt.getTime() - startedAt.getTime(); + call.metadata = normalizeReplayMetadata( + getReplayMetadataFromError(error), + ); + events.push({ + type: "tool_result", + toolCallId, + name: toolName, + error: serializeToolCallError(error), + startedAt: call.startedAt, + finishedAt: call.finishedAt, + durationMs: call.durationMs, + }); + throw error; + } + }, + ]), + ) as unknown as PiAiRuntime["tools"]; + + return { + tools: runtimeTools, + events: eventSink, + signal: context.signal, + }; +} + +function resolveOutput(result: unknown): JsonValue | undefined { + if (!result || typeof result !== "object") { + return undefined; + } + + const candidates = ["output"] satisfies string[]; + const resultRecord = result as Record; + + for (const key of candidates) { + if (Object.prototype.hasOwnProperty.call(resultRecord, key)) { + return toOutputValue(resultRecord[key]); + } + } + + return undefined; +} + +// Keep this adapter-local rather than exporting a core helper. Core +// `toJsonValue` normalizes trace data; inferred app output should only accept +// values that are already JSON-safe so the public contract stays explicit. +// Invalid nested values reject the whole candidate instead of being patched. +function toOutputValue(value: unknown): JsonValue | undefined { + if (value === undefined) { + return undefined; + } + + if ( + value === null || + typeof value === "string" || + typeof value === "boolean" + ) { + return value; + } + + if (typeof value === "number") { + return Number.isFinite(value) ? value : undefined; + } + + if (Array.isArray(value)) { + const items: JsonValue[] = []; + for (const item of value) { + const normalized = toOutputValue(item); + if (normalized === undefined) { + return undefined; + } + items.push(normalized); + } + return items; + } + + if (typeof value === "object") { + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + return undefined; + } + + const record: Record = {}; + for (const [key, entry] of Object.entries(value)) { + const normalized = toOutputValue(entry); + if (normalized === undefined) { + return undefined; + } + record[key] = normalized; + } + return record; + } + + return undefined; +} + +function normalizeToolResult(result: unknown): JsonValue | undefined { + const details = + result && typeof result === "object" + ? toJsonValue((result as { details?: unknown }).details) + : undefined; + if (details !== undefined) { + return details; + } + + const normalized = toJsonValue(result); + return normalized !== undefined + ? normalized + : result === undefined + ? undefined + : String(result); +} + +function normalizeReplayToolResult(result: unknown): JsonValue { + return normalizeToolResult(result) ?? null; +} + +type NativeToolReplayEnvelope = { + __vitestEvals: { + kind: "pi-ai-native-tool-result"; + version: 2; + }; + agentResult: JsonValue; + normalizedResult: JsonValue; +}; + +type LegacyNativeToolReplayEnvelope = { + __vitestEvals: { + kind: "pi-ai-native-tool-result"; + version: 1; + }; + normalizedResult: JsonValue; + agentResult?: JsonValue; +}; + +function createNativeToolReplayEnvelope( + result: unknown, +): NativeToolReplayEnvelope { + const normalizedResult = normalizeReplayToolResult(result); + + const agentResult = toJsonValue(result); + + return { + __vitestEvals: { + kind: "pi-ai-native-tool-result", + version: 2, + }, + agentResult: agentResult !== undefined ? agentResult : normalizedResult, + normalizedResult, + }; +} + +function resolveNativeToolReplayResult(result: JsonValue) { + if (isNativeToolReplayEnvelope(result)) { + return { + result: result.agentResult, + normalizedResult: result.normalizedResult, + }; + } + + if (isLegacyNativeToolReplayEnvelope(result)) { + return { + result: result.agentResult ?? result.normalizedResult, + normalizedResult: result.normalizedResult, + }; + } + + return { + result, + normalizedResult: normalizeReplayToolResult(result), + }; +} + +function isNativeToolReplayEnvelope( + value: unknown, +): value is NativeToolReplayEnvelope { + return Boolean( + value && + typeof value === "object" && + "__vitestEvals" in value && + isNativeToolReplayMarker( + (value as { __vitestEvals?: unknown }).__vitestEvals, + ) && + "agentResult" in value && + "normalizedResult" in value, + ); +} + +function isNativeToolReplayMarker( + value: unknown, +): value is NativeToolReplayEnvelope["__vitestEvals"] { + return Boolean( + value && + typeof value === "object" && + "kind" in value && + (value as { kind?: unknown }).kind === "pi-ai-native-tool-result" && + "version" in value && + (value as { version?: unknown }).version === 2, + ); +} + +function isLegacyNativeToolReplayEnvelope( + value: unknown, +): value is LegacyNativeToolReplayEnvelope { + return Boolean( + value && + typeof value === "object" && + "__vitestEvals" in value && + isLegacyNativeToolReplayMarker( + (value as { __vitestEvals?: unknown }).__vitestEvals, + ) && + "normalizedResult" in value, + ); +} + +function isLegacyNativeToolReplayMarker( + value: unknown, +): value is LegacyNativeToolReplayEnvelope["__vitestEvals"] { + return Boolean( + value && + typeof value === "object" && + "kind" in value && + (value as { kind?: unknown }).kind === "pi-ai-native-tool-result" && + "version" in value && + (value as { version?: unknown }).version === 1, + ); +} + +function countToolCallEvents(events: readonly TranscriptEvent[]) { + return events.filter((event) => event.type === "tool_call").length; +} + +function resolveUsage(result: unknown, toolCallCount: number): UsageSummary { + if (!result || typeof result !== "object") { + return toolCallCount > 0 ? { toolCalls: toolCallCount } : {}; + } + + const usageValue = + (result as Record).usage ?? + (result as Record).metrics; + + if ( + !usageValue || + typeof usageValue !== "object" || + Array.isArray(usageValue) + ) { + return toolCallCount > 0 ? { toolCalls: toolCallCount } : {}; + } + + const usageRecord = usageValue as Record; + const usage: UsageSummary = { + provider: stringField(usageRecord.provider), + model: stringField(usageRecord.model), + inputTokens: numberField(usageRecord.inputTokens), + outputTokens: numberField(usageRecord.outputTokens), + reasoningTokens: numberField(usageRecord.reasoningTokens), + totalTokens: numberField(usageRecord.totalTokens), + toolCalls: numberField(usageRecord.toolCalls), + retries: numberField(usageRecord.retries), + metadata: collectUsageMetadata(usageRecord), + }; + + if (usage.toolCalls === undefined && toolCallCount > 0) { + usage.toolCalls = toolCallCount; + } + + return usage; +} + +function collectUsageMetadata(usage: Record) { + const metadata: Record = {}; + const nestedMetadata = usage.metadata; + if ( + nestedMetadata && + typeof nestedMetadata === "object" && + !Array.isArray(nestedMetadata) + ) { + Object.assign(metadata, nestedMetadata); + } + + for (const [key, value] of Object.entries(usage)) { + if (key === "metadata" || USAGE_SUMMARY_KEYS.has(key)) { + continue; + } + metadata[key] = value; + } + + return normalizeMetadata(metadata); +} + +const USAGE_SUMMARY_KEYS = new Set([ + "provider", + "model", + "inputTokens", + "outputTokens", + "reasoningTokens", + "totalTokens", + "toolCalls", + "retries", +]); + +function stringField(value: unknown) { + return typeof value === "string" ? value : undefined; +} + +function numberField(value: unknown) { + return typeof value === "number" && Number.isFinite(value) + ? value + : undefined; +} + +function resolveSession( + result: unknown, + events: TranscriptEvent[], + output: JsonValue | undefined, + usage: UsageSummary, +): NormalizedSession { + if ( + isNormalizedSession( + (result as Record | undefined)?.session, + ) + ) { + return (result as { session: NormalizedSession }).session; + } + + const sessionEvents = [...events]; + if ( + output !== undefined && + !sessionEvents.some( + (event) => + event.type === "message" && + event.role === "assistant" && + event.content !== undefined, + ) + ) { + sessionEvents.push({ + type: "message", + role: "assistant", + content: output, + }); + } + + return { + events: sessionEvents, + provider: + ((result as Record | undefined)?.provider as + | string + | undefined) ?? usage.provider, + model: + ((result as Record | undefined)?.model as + | string + | undefined) ?? usage.model, + }; +} + +function resolveErrors(result: unknown): Array> { + return resolveHarnessRunErrors(result); +} + +async function executeToolWithReplay< + TArgs extends Record, + TResult extends JsonValue, + TInput, +>({ + toolName, + tool, + replay, + args, + context, +}: { + toolName: string; + tool: PiAiToolDefinition; + replay: PiAiToolReplayPolicy | undefined; + args: TArgs; + context: PiAiToolContext; +}) { + return executeWithReplay< + Record, + JsonValue, + PiAiToolContext + >({ + toolName, + args, + context, + execute: (toolArgs, toolContext) => + tool.execute(toolArgs as TArgs, toolContext), + replay, + }); +} diff --git a/packages/evals/src/general-knowledge.eval.ts b/packages/evals/src/smoke.eval.ts similarity index 54% rename from packages/evals/src/general-knowledge.eval.ts rename to packages/evals/src/smoke.eval.ts index 7354bb4ec..b00e586ed 100644 --- a/packages/evals/src/general-knowledge.eval.ts +++ b/packages/evals/src/smoke.eval.ts @@ -1,9 +1,14 @@ import { expect } from "vitest"; import { describeEval } from "vitest-evals"; -import { piCodingAgentHarness } from "./pi-harness.ts"; +import { createPiEvalAgent } from "./pi-agent.ts"; +import { piAiHarness } from "./pi-harness.ts"; -describeEval("general knowledge", { harness: piCodingAgentHarness }, (it) => { - it("knows the capital of France", async ({ run }) => { +const piCodingAgentHarness = piAiHarness({ + agent: ({ context }) => createPiEvalAgent({ noTools: "all" }, context.signal), +}); + +describeEval("Pi Coding Agent smoke", { harness: piCodingAgentHarness }, (it) => { + it("runs a basic prompt end to end", async ({ run }) => { const result = await run("What's the capital of France? Respond with only the city name."); expect(result.output.trim()).toBe("Paris");