diff --git a/README.md b/README.md index 83a1ff8ba..c90ba28d0 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ To learn more about Pi: | Package | Description | |---------|-------------| +| **[@earendil-works/pi-telemetry](packages/telemetry)** | Vendor-neutral telemetry contracts and typed schema utilities | | **[@earendil-works/pi-ai](packages/ai)** | Unified multi-provider LLM API (OpenAI, Anthropic, Google, etc.) | | **[@earendil-works/pi-agent-core](packages/agent)** | Agent runtime with tool calling and state management | | **[@earendil-works/pi-coding-agent](packages/coding-agent)** | Interactive coding agent CLI | diff --git a/package-lock.json b/package-lock.json index 3f954c459..9eadfcca2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -814,6 +814,10 @@ "resolved": "packages/session-backends/sqlite-node", "link": true }, + "node_modules/@earendil-works/pi-telemetry": { + "resolved": "packages/telemetry", + "link": true + }, "node_modules/@earendil-works/pi-tui": { "resolved": "packages/tui", "link": true @@ -5429,6 +5433,7 @@ "license": "MIT", "dependencies": { "@earendil-works/pi-ai": "^0.83.0", + "@earendil-works/pi-telemetry": "^0.83.0", "diff": "8.0.4", "ignore": "7.0.5", "typebox": "1.3.7", @@ -5475,6 +5480,7 @@ "dependencies": { "@anthropic-ai/sdk": "0.91.1", "@aws-sdk/client-bedrock-runtime": "3.1048.0", + "@earendil-works/pi-telemetry": "^0.83.0", "@google/genai": "1.52.0", "@mistralai/mistralai": "2.2.6", "@opentelemetry/api": "1.9.0", @@ -5751,6 +5757,35 @@ "node": ">=22.19.0" } }, + "packages/telemetry": { + "name": "@earendil-works/pi-telemetry", + "version": "0.83.0", + "license": "MIT", + "devDependencies": { + "@types/node": "24.12.4", + "vitest": "4.1.9" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "packages/telemetry/node_modules/@types/node": { + "version": "24.12.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.4.tgz", + "integrity": "sha512-GUUEShf+PBCGW2KaXwcIt3Yk+e3pkKwWKb9GSyM9WQVE+ep2jzmHdGsHzu4wgcZy5fN9FBdVzjpBQsYlpfpgLA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "packages/telemetry/node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "dev": true, + "license": "MIT" + }, "packages/tui": { "name": "@earendil-works/pi-tui", "version": "0.83.0", diff --git a/package.json b/package.json index 6d2e92328..a417480ec 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,8 @@ ], "scripts": { "clean": "npm run clean --workspaces", - "build": "cd packages/tui && npm run build && cd ../ai && npm run build && cd ../agent && npm run build && cd ../session-backends/sqlite-node && npm run build && cd ../../protocol && npm run build && cd ../client && npm run build && cd ../server && npm run build && cd ../coding-agent && npm run build", - "build:offline": "cd packages/tui && npm run build && cd ../ai && npm run build:offline && cd ../agent && npm run build && cd ../session-backends/sqlite-node && npm run build && cd ../../protocol && npm run build && cd ../client && npm run build && cd ../server && npm run build && cd ../coding-agent && npm run build", + "build": "cd packages/tui && npm run build && cd ../telemetry && npm run build && cd ../ai && npm run build && cd ../agent && npm run build && cd ../session-backends/sqlite-node && npm run build && cd ../../protocol && npm run build && cd ../client && npm run build && cd ../server && npm run build && cd ../coding-agent && npm run build", + "build:offline": "cd packages/tui && npm run build && cd ../telemetry && npm run build && cd ../ai && npm run build:offline && cd ../agent && npm run build && cd ../session-backends/sqlite-node && npm run build && cd ../../protocol && npm run build && cd ../client && npm run build && cd ../server && npm run build && cd ../coding-agent && npm run build", "check": "biome check --write --error-on-warnings . && npm run check:pinned-deps && npm run check:ts-imports && npm run check:shrinkwrap && npm run check:install-lock:coding-agent && tsgo --noEmit && npm run check:browser-smoke", "check:browser-smoke": "node scripts/check-browser-smoke.mjs", "check:pinned-deps": "node scripts/check-pinned-deps.mjs", diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 457a6d75b..7241660f4 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -10,7 +10,7 @@ ### Added -- Added the typed harness telemetry schema, callback helpers, and generated schema reference. +- Added typed AI-request and harness telemetry schemas, callback helpers, and a generated schema reference. - Added bounded `Session.findEntriesOnBranch()` and `findEntryOnBranch()` queries with explicit traversal, filtering, ordering, and limit options. - Added a compile-complete `AgentHarness` v2 scaffold; unfinished operation paths reject with `HarnessNotImplemented` while durable execution is implemented. - Added `JsonlSessionRepo`, a v4 append-only JSONL session repository with metadata validation and shared storage semantics ([#7611](https://github.com/earendil-works/pi/pull/7611) by [@davidbrai](https://github.com/davidbrai)). diff --git a/packages/agent/docs/harness-v2.md b/packages/agent/docs/harness-v2.md index 4de697fdd..f9a6a1095 100644 --- a/packages/agent/docs/harness-v2.md +++ b/packages/agent/docs/harness-v2.md @@ -4,7 +4,7 @@ > **Compatibility policy.** Old coding-agent v3 JSONL sessions must open and restore idle. This is the only backward-compatibility requirement. All other formats and APIs in `packages/agent/src/harness` and `packages/session-backends/sqlite-node` (and their respective tests) may break. We do not write migrations, schema versioning, or conversion paths for anything else. -> **Implementation-status convention.** Sections 1–20 describe the chosen end state unless a paragraph explicitly says that behavior is landed; their code blocks are target contracts, not claims that the current scaffold implements them. Section 21 is the source of truth. At the I0 landing (`04d6447f7`), the callback contract, schemas, typed helpers, no-op context, request-option propagation, exports, generators, and generated references are landed. Section 16’s provider/deferred APIs are also landed, but its settled agent-side narrowing and harness integration are future work. The section 14 public building blocks, canonical `AgentHarnessOptions.telemetryContext`, effect-context threading, and all runtime span creation remain future L/H/I/O work; the scaffold still has its temporary `context?: ExecutionContext` surface. +> **Implementation-status convention.** Sections 1–20 describe the chosen end state unless a paragraph explicitly says that behavior is landed; their code blocks are target contracts, not claims that the current scaffold implements them. Section 21 is the source of truth. The callback contract, schemas, typed helpers, no-op context, request-option propagation, exports, generator, and generated reference are landed. Section 16’s provider/deferred APIs are also landed, but its settled agent-side narrowing and harness integration are future work. The section 14 public building blocks, canonical `AgentHarnessOptions.telemetryContext`, effect-context threading, and all runtime span creation remain future L/H/I/O work; the scaffold still has its temporary `context?: ExecutionContext` surface. ```mermaid flowchart TD @@ -42,7 +42,7 @@ The harness executes runs against one session. The session holds four kinds of s - **Provider stream resumption.** Partial streams are never persisted. An interrupted streaming request is retried or abandoned. Deferred requests are different and in scope: the provider returns a handle at once and serves the result later (e.g. `background: true` on a Responses API, batch APIs). pi-ai returns an assistant message with stop reason `deferred` that carries the handle; it is persisted like any assistant message. Redeeming the handle appends a normal assistant message. Recovery sees the unredeemed handle and fetches instead of paying for a new request. - **Multiple writers.** Two processes on one session are out of scope. The serving layer routes all traffic for a session to the process that holds its harness. Lanes cover the workloads that look like multi-writer: parallel threads over shared history. - **Replication.** A session lives in one place. Coordination-free sync of diverging copies is a different design. Nothing forecloses it later; section 19 records why. -- **Coding-agent migration.** `packages/coding-agent` remains on its current runtime and is not modified by this implementation plan. Compatibility means the new JSONL repository can read supported coding-agent v3 files; it does not mean coding-agent is switched to `AgentHarness` here. +- **Coding-agent migration.** `packages/coding-agent` remains on its current runtime; only telemetry dependency build ordering and generated locks may change. Compatibility means the new JSONL repository can read supported coding-agent v3 files; it does not mean coding-agent is switched to `AgentHarness` here. ## 2. What a session is @@ -1047,7 +1047,7 @@ Calls on a faulted harness reject with the same `HarnessFault` instance until th `finalMessage` is the run's newest entry that projects to an assistant message; `finalEntryId` is that entry's id. `leafId` is the lane's leaf when the operation finished — the race-free anchor for branch queries (`findEntriesOnBranch({ start: leafId })`). The two differ when a deferred write was applied after the final assistant message. Full transcripts are not duplicated into results; they are in the session and were delivered as events. -**Type provenance.** Core conversation and tool types (`AgentMessage`, `AgentTool`, `AgentToolResult`, `QueueMode`, `ThinkingLevel`) come from `packages/agent/src/types.ts`. Provider types (`Model`, `Models`, `Usage`, `RetryPolicy`, stream options, deferred handles) come from `packages/ai`. The landed generic telemetry contract, schema machinery, and pi-ai span schema come from `packages/ai/src/telemetry.ts`; the landed harness span schema comes from `packages/agent/src/harness/telemetry.ts`. The current harness scaffold still declares its temporary local `ExecutionContext` / `ExecutionSpan` types and `context` option; H0 replaces those declarations with the canonical telemetry types and target `telemetryContext` option. Session, harness, hook, event, result, snapshot, navigation, and durable-record types are defined by the v2 implementation under `packages/agent/src/harness/`. Lowercase helpers in section 15 pseudocode without a definition (`preparation`, `runToolBatchForSingleCall`, request/option bags such as `AssistantRequest` and `FactWrite`) are constructive implementation detail, not contract. +**Type provenance.** Core conversation and tool types (`AgentMessage`, `AgentTool`, `AgentToolResult`, `QueueMode`, `ThinkingLevel`) come from `packages/agent/src/types.ts`. Provider types (`Model`, `Models`, `Usage`, `RetryPolicy`, stream options, deferred handles) come from `packages/ai`. The landed generic telemetry contract and schema machinery come from `packages/telemetry/src/index.ts`; the landed AI-request and harness span schemas come from `packages/agent/src/harness/telemetry.ts`. The current harness scaffold still declares its temporary local `ExecutionContext` / `ExecutionSpan` types and `context` option; H0 replaces those declarations with the canonical telemetry types and target `telemetryContext` option. Session, harness, hook, event, result, snapshot, navigation, and durable-record types are defined by the v2 implementation under `packages/agent/src/harness/`. Lowercase helpers in section 15 pseudocode without a definition (`preparation`, `runToolBatchForSingleCall`, request/option bags such as `AssistantRequest` and `FactWrite`) are constructive implementation detail, not contract. ### Suspended operations @@ -2763,7 +2763,7 @@ repo.create({ id?, parentSessionId? }): Promise; ## 18. Telemetry -**Implementation status:** I0 landed the backend-neutral contract, no-op context, schema machinery, exact pi-ai and harness schemas, typed helpers, request-option propagation, exports, generators, references, and focused tests. No current agent-loop or harness execution path calls `startAiSpan()` or `startHarnessSpan()`, so I0 emits no runtime spans. H0/L1–L3/I4 will establish canonical context plumbing; O2 will insert the spans described under Effects, nesting, and lifetime below. +**Implementation status:** I0 and the telemetry-package extraction landed the backend-neutral contract, no-op context, schema machinery, exact AI-request and harness schemas, typed helpers, request-option propagation, exports, one generator/reference, and focused tests. No current agent-loop or harness execution path calls `startAiSpan()` or `startHarnessSpan()`, so I0 emits no runtime spans. H0/L1–L3/I4 will establish canonical context plumbing; O2 will insert the spans described under Effects, nesting, and lifetime below. Telemetry uses explicit context propagation. Core code does not use `AsyncLocalStorage`, global current-span state, or runtime-specific context APIs: pi runs in Node, Bun, browsers, and workers, so no runtime's ambient-context mechanism can be the core abstraction. An adapter may use ambient context internally — for example, an OpenTelemetry adapter may activate its native child context so HTTP auto-instrumentation attaches correctly — but pi always passes the parent explicitly. @@ -2771,9 +2771,9 @@ Pi ships no exporter and requires no backend-specific telemetry implementation. ### Package ownership -The generic contract, schema-definition machinery, shared no-op implementation, and pi-ai span schema live in `packages/ai/src/telemetry.ts` and are exported from `@earendil-works/pi-ai`. `packages/agent/src/harness/telemetry.ts` imports the canonical generic types, re-exports the generic contract/schema types it exposes, and adds the harness span schema and inferred harness types. The agent package root explicitly re-exports the complete pi-ai telemetry surface plus the harness schema and helpers. There is one generic contract, never one copy per package. +The generic contract, schema-definition machinery, and shared no-op implementation live in `packages/telemetry/src/index.ts` and are exported from `@earendil-works/pi-telemetry`. Pi-ai imports only `TelemetryContext` for request options; it owns no span schema or helper and emits no telemetry itself. `packages/agent/src/harness/telemetry.ts` owns both `AI_TELEMETRY_SCHEMA` / `startAiSpan()` and `HARNESS_TELEMETRY_SCHEMA` / `startHarnessSpan()`. The agent package root re-exports those domain schemas and helpers plus the generic telemetry surface. There is one generic contract and one current domain-schema owner. -The landed `ProviderRequestOptions.telemetryContext` carries an optional parent through stream/simple-stream, deferred fetch/cancel, and image request options; provider, `Models`, `ImagesModels`, direct dispatch, and built-in simple-option conversion preserve the object unchanged. The current `AgentHarnessOptions` still has its temporary `context?: ExecutionContext` field. H0 will replace that scaffold surface with `telemetryContext?: TelemetryContext`, L/I/H work will thread it, and O2 will make the agent-side request wrapper emit `pi.ai.request` through the pi-ai schema. Pi-ai owns the vocabulary and helper, not a second ambient runtime. +The landed `ProviderRequestOptions.telemetryContext` carries an optional parent through stream/simple-stream, deferred fetch/cancel, and image request options; provider, `Models`, `ImagesModels`, direct dispatch, and built-in simple-option conversion preserve the object unchanged. This leaves pi-ai API implementations free to use the context later without making pi-ai own the generic API today. The current `AgentHarnessOptions` still has its temporary `context?: ExecutionContext` field. H0 will replace that scaffold surface with `telemetryContext?: TelemetryContext`, L/I/H work will thread it, and O2 will make the agent-side request wrapper emit `pi.ai.request` through the agent-owned AI schema. Both schemas are pi-owned. Span names use the `pi.ai.*`, `pi.harness.*`, and `pi.session.*` families; attributes use the same pi-owned `pi.*` vocabulary and do not adopt an external semantic-convention namespace. Adapters translate them when useful; the emitted pi vocabulary remains stable regardless of backend convention churn. @@ -2815,7 +2815,7 @@ interface TelemetrySpan extends TelemetryContext { } ``` -I0 exports the shared no-op context from pi-ai. The target harness and L3 compatibility wrapper will select it when no application context is supplied; the current scaffold does not yet apply that default. Under the context contract, `startSpan()` creates the child and invokes its callback synchronously, exactly once, before returning a promise. It keeps the span open until the callback's value or promise settles: +The telemetry package exports the shared no-op context. The target harness and L3 compatibility wrapper will select it when no application context is supplied; the current scaffold does not yet apply that default. Under the context contract, `startSpan()` creates the child and invokes its callback synchronously, exactly once, before returning a promise. It keeps the span open until the callback's value or promise settles: - return or resolve: default status `ok`, then automatic end; - synchronous throw: return a promise rejected with the same thrown value, after automatic error status and end; @@ -2842,7 +2842,7 @@ A `TelemetrySpan` is also the explicit child `TelemetryContext`. Passing the cal ### Typed schema -The low-level adapter accepts the open `SpanAttributes` bag. O2 instrumentation must never construct untyped span names or attribute bags directly. I0 already exports one plain, serializable schema object and typed helpers per package for that purpose. +The low-level adapter accepts the open `SpanAttributes` bag. O2 instrumentation must never construct untyped span names or attribute bags directly. The agent package already exports the two plain, serializable domain schema objects and their typed helpers for that purpose. ```ts type TelemetryAttributeType = @@ -2905,7 +2905,7 @@ declare function defineTelemetrySchema = InferStartAttributes< @@ -2982,7 +2982,7 @@ type AiSpanAttributes = AiSpanStartAttributes The following tables are normative input to the schema objects. `!` means a required start attribute; `?` means an optional start attribute. Every end attribute is optional enrichment. Array element sets use `elementValues`; all other closed sets use `values`. The automatic throw/reject rule from the context contract applies to every span in addition to the explicit status rule shown. -#### Pi-ai schema +#### AI request schema `AI_TELEMETRY_SCHEMA` declares no pi-written span events and one span. Its parent rule is `{ kind: "any" }`: @@ -3040,9 +3040,9 @@ The parent column maps directly to `TelemetryParentDefinition`: “root or appli Dynamic identifiers and names are attributes, never span names. The landed schema definitions are the exhaustive vocabulary O2 may emit; I0 itself emits no pi-written runtime span, event, or attribute. -Each package exports its schema, span-name union, per-name start/end/combined attribute types, event types, discriminated span union, and a typed `startAiSpan()` or `startHarnessSpan()` helper. A helper accepts only that span's start attributes; its callback receives a schema-scoped view of the live span whose `setAttributes()` accepts only that span's optional end attributes and whose `addEvent()` accepts only declared event names and attributes. Individual calls reject missing required start attributes, unknown attributes, type mismatches, and invalid closed-set values at compile time. TypeScript does not try to prove that any end setter ran; `startSpan()` always owns automatic settlement. The scoped view erases to the generic `TelemetrySpan`; production performs no schema validation. +The agent package exports both schemas, each span-name union, per-name start/end/combined attribute types, event types, discriminated span unions, and typed `startAiSpan()` / `startHarnessSpan()` helpers. A helper accepts only that span's start attributes; its callback receives a schema-scoped view of the live span whose `setAttributes()` accepts only that span's optional end attributes and whose `addEvent()` accepts only declared event names and attributes. Individual calls reject missing required start attributes, unknown attributes, type mismatches, and invalid closed-set values at compile time. TypeScript does not try to prove that any end setter ran; `startSpan()` always owns automatic settlement. The scoped view erases to the generic `TelemetrySpan`; production performs no schema validation. -The schema objects are also the documentation source. I0 added `packages/ai/scripts/generate-telemetry-docs.ts` and `packages/agent/scripts/generate-telemetry-docs.ts`, with package scripts `generate-telemetry-docs` and `check:telemetry-docs`, to generate `packages/ai/docs/telemetry-schema.md` and `packages/agent/docs/telemetry-schema.md`. The Markdown files are repository documentation, not npm package files; published consumers import the serializable schema objects from each package root. Schema `version` starts at 1; package changelogs record compatible additions and breaking renames, removals, type changes, or meaning changes. Explicit migration metadata is added only if a real consumer needs automatic translation. +The schema objects are also the documentation source. `packages/agent/scripts/generate-telemetry-docs.ts`, exposed through package scripts `generate-telemetry-docs` and `check:telemetry-docs`, generates the combined AI-request and harness reference at `packages/agent/docs/telemetry-schema.md`. The Markdown file is repository documentation, not an npm package file; published consumers import both serializable schema objects from the agent package root. Schema `version` starts at 1; package changelogs record compatible additions and breaking renames, removals, type changes, or meaning changes. Explicit migration metadata is added only if a real consumer needs automatic translation. ### Effects and nesting @@ -3222,7 +3222,7 @@ Gate invariants, asserted across Tier C: ## 21. Implementation status and work packages -Harness implementation lives directly in `packages/agent/src/harness/`, with v4 session tests in `packages/agent/test/harness/session/`. The generic telemetry contract and pi-ai schema live in `packages/ai/src/telemetry.ts`. The v4 session and SQLite backend are the default package interfaces; there is no experimental package surface. Retained compaction, message projection, resource, tool, environment, and utility modules stay under `src/harness/` and are adapted in place. +Harness implementation lives directly in `packages/agent/src/harness/`, with v4 session tests in `packages/agent/test/harness/session/`. The generic telemetry contract lives in `packages/telemetry/src/index.ts`; both current domain schemas live in `packages/agent/src/harness/telemetry.ts`. The v4 session and SQLite backend are the default package interfaces; there is no experimental package surface. Retained compaction, message projection, resource, tool, environment, and utility modules stay under `src/harness/` and are adapted in place. ### Landed foundation @@ -3233,11 +3233,11 @@ Harness implementation lives directly in `packages/agent/src/harness/`, with v4 - Compaction preparation, context projection, and branch summarization use v4 `Entry` queries and no longer depend on the legacy `SessionTreeEntry` model. Reusable compaction tests and dedicated context tests cover this interim behavior. - `Session.getBranch()` is not part of v4 and must not be reintroduced. Branch callers use `findEntriesOnBranch()` with an explicit start and deliberate ordering semantics; callers may use the documented `newestFirst` default. - The pi-ai deferred request types, `Models.fetchDeferred()` / `cancelDeferred()` dispatch, and faux-provider pending/ready/terminal/cancellation support are already implemented and tested. Only harness integration remains. -- The canonical telemetry contract, exact typed pi-ai and harness schemas, no-op context, request-option propagation (including built-in simple-option conversion), generated references, and focused tests are implemented. The current `AgentHarness` scaffold still uses temporary local execution-context types; canonical harness option/threading is H0/L/I work, and runtime span insertion remains O2. +- The canonical telemetry contract, exact typed AI-request and harness schemas, no-op context, request-option propagation (including built-in simple-option conversion), generated reference, and focused tests are implemented. The current `AgentHarness` scaffold still uses temporary local execution-context types; canonical harness option/threading is H0/L/I work, and runtime span insertion remains O2. ### Scope boundary -`packages/coding-agent/**` remains untouched. This plan implements `packages/agent` and `packages/session-backends/sqlite-node`, plus the generic telemetry contract, pi-ai schema, exports, tests, and generated schema documentation under `packages/ai`. O2 consumes that pi-ai telemetry surface but otherwise keeps provider behavior unchanged. The v3 requirement is an input-format requirement for the new JSONL repository, not a coding-agent migration. No work package may modify coding-agent source, tests, RPC, UI, or package metadata. +Coding-agent source and runtime behavior remain untouched; its binary build order and generated dependency locks reflect the telemetry dependency. This plan implements `packages/agent` and `packages/session-backends/sqlite-node`, plus the generic telemetry package and pi-ai request-option propagation. O2 consumes the agent-owned AI telemetry schema but otherwise keeps provider behavior unchanged. The v3 requirement is an input-format requirement for the new JSONL repository, not a coding-agent migration. No work package may modify coding-agent source, tests, RPC, UI, or package metadata except I0's telemetry build-order integration. ### Package rules @@ -3376,10 +3376,12 @@ These packages own `packages/agent/src/harness/session/jsonl/**`, the concrete ` I0, I1, and I2 may proceed independently. I3 → I4 → I5 is serial and begins after R2 fixes the `LaneState` shape. These packages use separate modules with focused unit tests; I5 remains primitive-only and does not edit `agent-harness.ts`. - [x] **I0 — telemetry contracts, typed schemas, and no-op context.** Dependencies: none. - - Primary files: `packages/ai/src/telemetry.ts`, `packages/ai/src/index.ts`, focused tests, package scripts, `packages/ai/scripts/generate-telemetry-docs.ts`, and generated `packages/ai/docs/telemetry-schema.md`; `packages/agent/src/harness/telemetry.ts`, `packages/agent/src/index.ts`, focused tests, package scripts, `packages/agent/scripts/generate-telemetry-docs.ts`, and generated `packages/agent/docs/telemetry-schema.md`. Do not edit `agent-harness.ts`; H0 replaces its temporary local type declarations after convergence. - - In pi-ai, implement the one canonical section 18 callback-based `TelemetryContext` / `TelemetrySpan` contract, shared no-op context, serializable `defineTelemetrySchema()` machinery, the complete normative `AI_TELEMETRY_SCHEMA`, inferred span-name/attribute/union types, and typed `startAiSpan()` helper. Add optional `telemetryContext` to `ProviderRequestOptions` so every stream, deferred, and image option inherits it; provider, `Models`, `ImagesModels`, and direct dispatch preserve it. Export the telemetry surface from `@earendil-works/pi-ai`. - - In agent, import and re-export the generic pi-ai surface; define the complete normative `HARNESS_TELEMETRY_SCHEMA`, inferred harness types, and typed `startHarnessSpan()` helper, and export the module from the agent package root. Do not duplicate the generic contract and do not adopt OTel or another external semantic convention. - - Generate the two repository-only Markdown references from the runtime schema values with the named package scripts. Production helpers perform no runtime schema validation; schemas compile-time-check each pi-written start/end/event call and remain importable as machine-readable data. + - Primary files: `packages/telemetry/src/index.ts` and focused tests; pi-ai request-option types/propagation and focused tests; `packages/agent/src/harness/telemetry.ts`, `packages/agent/src/index.ts`, focused tests, package scripts, `packages/agent/scripts/generate-telemetry-docs.ts`, and generated `packages/agent/docs/telemetry-schema.md`. Do not edit `agent-harness.ts`; H0 replaces its temporary local type declarations after convergence. + - In telemetry, implement the one canonical section 18 callback-based `TelemetryContext` / `TelemetrySpan` contract, shared no-op context, and serializable `defineTelemetrySchema()` machinery. + - In pi-ai, add optional `telemetryContext` to `ProviderRequestOptions` so every stream, deferred, and image option inherits it; provider, `Models`, `ImagesModels`, direct dispatch, and simple-option conversion preserve it. Pi-ai owns no domain schema or helper. + - In agent, define the complete normative `AI_TELEMETRY_SCHEMA` and `HARNESS_TELEMETRY_SCHEMA`, their inferred types, and typed `startAiSpan()` / `startHarnessSpan()` helpers. Export both schemas/helpers and re-export the generic telemetry surface from the agent package root. Do not duplicate the generic contract and do not adopt OTel or another external semantic convention. + - Generate the combined repository-only Markdown reference from the runtime schema values with the named agent package scripts. Production helpers perform no runtime schema validation; schemas compile-time-check each pi-written start/end/event call and remain importable as machine-readable data. + - Wire telemetry before pi-ai in workspace, local-release, publish, profiling, and coding-agent binary build order; add source-test aliases and refresh workspace/generated dependency locks. - Landed coverage: focused tests exercise no-op synchronous admission, returned-value and sync/async rejection preservation, explicit no-op child propagation, one shared frozen inert span with no payload inspection, exact start/optional-end inference, rejection of missing, unknown, empty-schema, and invalid closed-set attributes, absence of declared span events, schema JSON serialization, option propagation across provider/`Models` stream and deferred dispatch, direct and `ImagesModels` image dispatch, built-in simple-option conversion, and generated-document freshness. Pi ships no recording adapter; section 18's merge/status/post-settlement rules are adapter obligations, while O2 will test pi's runtime status and nesting behavior with captured spans. - [ ] **I1 — hook registry and runner.** Dependencies: none. - Primary files: `packages/agent/src/harness/hooks.ts`, `packages/agent/test/harness/hooks.test.ts`. @@ -3500,8 +3502,8 @@ For a fresh implementation session, in this order. This document wins over older 5. `packages/session-backends/sqlite-node/src/sqlite/repo.ts` — v4 SQLite repository, leases, and forks. 6. `packages/session-backends/sqlite-node/src/sqlite/storage/branch-entries.ts` — branch cache queries. 7. `packages/agent/src/harness/agent-harness.ts` — v2 public API scaffold, including the temporary local execution-context declarations that H0 replaces. -8. `packages/ai/src/telemetry.ts` — landed I0 canonical telemetry contract, no-op context, schema machinery, pi-ai schema, and typed helper; no runtime span insertion. -9. `packages/agent/src/harness/telemetry.ts` — landed I0 harness schema and typed helper; no harness instrumentation yet. +8. `packages/telemetry/src/index.ts` — canonical telemetry contract, no-op context, and schema machinery. +9. `packages/agent/src/harness/telemetry.ts` — landed AI-request and harness schemas and typed helpers; no runtime instrumentation yet. 10. `packages/agent/src/agent-loop.ts` — the still-legacy loop that L1–L3 will split into the section 14 public building blocks. 11. `packages/agent/src/agent.ts` — queues, continuation, abort, settlement to preserve in spirit. 12. `packages/agent/src/harness/messages.ts` — message conversion (`toProviderMessages` default). diff --git a/packages/agent/docs/telemetry-schema.md b/packages/agent/docs/telemetry-schema.md index be32446bd..9eb8e6e4e 100644 --- a/packages/agent/docs/telemetry-schema.md +++ b/packages/agent/docs/telemetry-schema.md @@ -1,10 +1,60 @@ -# Pi Agent Harness Telemetry Schema +# Pi Agent Telemetry Schemas +## AI request schema + Schema version: 1 -## `pi.harness.run` +### `pi.ai.request` + +One logical request to an AI provider + +- Parents: root or any caller span +- Default status: `ok` +- Error when: The operation throws or returns an error result + +#### Start attributes + +| Name | Type | Required | Values | Notes | Description | +|---|---|---:|---|---|---| +| `pi.ai.operation` | `string` | yes | stream, fetch_deferred, cancel_deferred, generate_images | | Logical provider operation | +| `pi.ai.provider` | `string` | yes | | | Selected provider id | +| `pi.ai.model` | `string` | yes | | | Requested model id | +| `pi.ai.api` | `string` | yes | | | Provider API id | +| `pi.ai.streaming` | `boolean` | yes | | | Whether this operation returns a stream | +| `pi.ai.deferred` | `boolean` | no | | | Whether the operation requests or participates in deferred execution | + +#### End attributes + +All end attributes are optional completion enrichment. + +| Name | Type | Values | Notes | Description | +|---|---|---|---|---| +| `pi.ai.response.model` | `string` | | | Concrete response model | +| `pi.ai.response.id` | `string` | | high cardinality | Provider response id | +| `pi.ai.response.stop_reason` | `string` | stop, length, tool_use, error, aborted, deferred | | Normalized terminal response reason | +| `pi.ai.http.status_code` | `number` | | | Final HTTP status | +| `pi.ai.usage.input_tokens` | `number` | | | Reported input tokens | +| `pi.ai.usage.output_tokens` | `number` | | | Reported output tokens | +| `pi.ai.usage.cache_read_tokens` | `number` | | | Reported cache-read tokens | +| `pi.ai.usage.cache_write_tokens` | `number` | | | Reported cache-write tokens | +| `pi.ai.usage.reasoning_tokens` | `number` | | | Reported reasoning tokens | +| `pi.ai.usage.total_tokens` | `number` | | | Reported total tokens | +| `pi.ai.usage.cost` | `number` | | | Reported total cost | +| `pi.ai.stream.chunk_count` | `number` | | | Streamed update chunk count | +| `pi.ai.stream.time_to_first_chunk_ms` | `number` | | | Elapsed milliseconds to first update chunk | +| `pi.ai.error.type` | `string` | | low cardinality | Provider or transport error class | + +#### Events + +No declared span events. + +## Harness schema + +Schema version: 1 + +### `pi.harness.run` One admitted in-process run invocation @@ -12,7 +62,7 @@ One admitted in-process run invocation - Default status: `ok` - Error when: The run fails or throws -### Start attributes +#### Start attributes | Name | Type | Required | Values | Notes | Description | |---|---|---:|---|---|---| @@ -22,7 +72,7 @@ One admitted in-process run invocation | `pi.operation.recovery` | `boolean` | yes | | | Whether this invocation resumes durable work | | `pi.operation.kind` | `string` | yes | run | | Run operation kind | -### End attributes +#### End attributes All end attributes are optional completion enrichment. @@ -32,11 +82,11 @@ All end attributes are optional completion enrichment. | `pi.error.code` | `string` | | low cardinality | Stable operation error code | | `pi.error.type` | `string` | | low cardinality | Low-cardinality operation error class | -### Events +#### Events No declared span events. -## `pi.harness.compaction` +### `pi.harness.compaction` One admitted in-process manual compaction invocation @@ -44,7 +94,7 @@ One admitted in-process manual compaction invocation - Default status: `ok` - Error when: The compaction fails or throws -### Start attributes +#### Start attributes | Name | Type | Required | Values | Notes | Description | |---|---|---:|---|---|---| @@ -54,7 +104,7 @@ One admitted in-process manual compaction invocation | `pi.operation.recovery` | `boolean` | yes | | | Whether this invocation resumes durable work | | `pi.operation.kind` | `string` | yes | compaction | | Compaction operation kind | -### End attributes +#### End attributes All end attributes are optional completion enrichment. @@ -64,11 +114,11 @@ All end attributes are optional completion enrichment. | `pi.error.code` | `string` | | low cardinality | Stable operation error code | | `pi.error.type` | `string` | | low cardinality | Low-cardinality operation error class | -### Events +#### Events No declared span events. -## `pi.harness.navigation` +### `pi.harness.navigation` One admitted in-process navigation invocation @@ -76,7 +126,7 @@ One admitted in-process navigation invocation - Default status: `ok` - Error when: The navigation fails or throws -### Start attributes +#### Start attributes | Name | Type | Required | Values | Notes | Description | |---|---|---:|---|---|---| @@ -86,7 +136,7 @@ One admitted in-process navigation invocation | `pi.operation.recovery` | `boolean` | yes | | | Whether this invocation resumes durable work | | `pi.operation.kind` | `string` | yes | navigation | | Navigation operation kind | -### End attributes +#### End attributes All end attributes are optional completion enrichment. @@ -96,11 +146,11 @@ All end attributes are optional completion enrichment. | `pi.error.code` | `string` | | low cardinality | Stable operation error code | | `pi.error.type` | `string` | | low cardinality | Low-cardinality operation error class | -### Events +#### Events No declared span events. -## `pi.harness.checkpoint` +### `pi.harness.checkpoint` One run checkpoint @@ -108,7 +158,7 @@ One run checkpoint - Default status: `ok` - Error when: Checkpoint work throws -### Start attributes +#### Start attributes | Name | Type | Required | Values | Notes | Description | |---|---|---:|---|---|---| @@ -116,7 +166,7 @@ One run checkpoint | `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id | | `pi.checkpoint.kind` | `string` | yes | normal, failure_drain, abort_reconcile | | Checkpoint purpose | -### End attributes +#### End attributes All end attributes are optional completion enrichment. @@ -124,11 +174,11 @@ All end attributes are optional completion enrichment. |---|---|---|---|---| | _none_ | | | | | -### Events +#### Events No declared span events. -## `pi.harness.turn` +### `pi.harness.turn` One assistant response and its tool batch @@ -136,7 +186,7 @@ One assistant response and its tool batch - Default status: `ok` - Error when: Turn work throws -### Start attributes +#### Start attributes | Name | Type | Required | Values | Notes | Description | |---|---|---:|---|---|---| @@ -144,7 +194,7 @@ One assistant response and its tool batch | `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id | | `pi.turn.id` | `string` | yes | | high cardinality | Invocation-local turn id | -### End attributes +#### End attributes All end attributes are optional completion enrichment. @@ -152,11 +202,11 @@ All end attributes are optional completion enrichment. |---|---|---|---|---| | _none_ | | | | | -### Events +#### Events No declared span events. -## `pi.harness.step` +### `pi.harness.step` One durable retry attempt @@ -164,7 +214,7 @@ One durable retry attempt - Default status: `ok` - Error when: The attempt retries, fails, or throws -### Start attributes +#### Start attributes | Name | Type | Required | Values | Notes | Description | |---|---|---:|---|---|---| @@ -174,7 +224,7 @@ One durable retry attempt | `pi.step.attempt` | `number` | yes | | | One-based durable attempt number | | `pi.compaction.reason` | `string` | no | manual, threshold, overflow | | Compaction trigger | -### End attributes +#### End attributes All end attributes are optional completion enrichment. @@ -182,11 +232,11 @@ All end attributes are optional completion enrichment. |---|---|---|---|---| | `pi.step.outcome` | `string` | succeeded, retry, failed, aborted, deferred, overflow | | Attempt outcome | -### Events +#### Events No declared span events. -## `pi.harness.tool` +### `pi.harness.tool` One raw phase-2 tool execution @@ -194,7 +244,7 @@ One raw phase-2 tool execution - Default status: `ok` - Error when: Raw phase-2 execution returns an error -### Start attributes +#### Start attributes | Name | Type | Required | Values | Notes | Description | |---|---|---:|---|---|---| @@ -206,7 +256,7 @@ One raw phase-2 tool execution | `pi.tool.replay` | `string` | yes | never, safe | | Declared replay policy | | `pi.tool.recovery` | `boolean` | yes | | | Whether this is recovery execution | -### End attributes +#### End attributes All end attributes are optional completion enrichment. @@ -214,11 +264,11 @@ All end attributes are optional completion enrichment. |---|---|---|---|---| | `pi.tool.is_error` | `boolean` | | | Whether raw phase-2 execution returned an error | -### Events +#### Events No declared span events. -## `pi.harness.hook` +### `pi.harness.hook` One registered hook handler invocation @@ -226,7 +276,7 @@ One registered hook handler invocation - Default status: `ok` - Error when: The handler throws -### Start attributes +#### Start attributes | Name | Type | Required | Values | Notes | Description | |---|---|---:|---|---|---| @@ -235,7 +285,7 @@ One registered hook handler invocation | `pi.hook.name` | `string` | yes | before_run, before_resume, before_run_end, transform_context, before_request, before_payload, after_response, before_tool, after_tool, before_compaction, before_navigation | | Hook name | | `pi.hook.registration_id` | `string` | no | | | Stable hook registration id | -### End attributes +#### End attributes All end attributes are optional completion enrichment. @@ -243,11 +293,11 @@ All end attributes are optional completion enrichment. |---|---|---|---|---| | `pi.hook.outcome` | `string` | completed, skipped, blocked, failed | | Handler outcome | -### Events +#### Events No declared span events. -## `pi.harness.sleep` +### `pi.harness.sleep` One retry delay @@ -255,14 +305,14 @@ One retry delay - Default status: `ok` - Error when: Sleep work throws -### Start attributes +#### Start attributes | Name | Type | Required | Values | Notes | Description | |---|---|---:|---|---|---| | `pi.operation.id` | `string` | yes | | high cardinality | Durable operation id | | `pi.sleep.delay_ms` | `number` | yes | | | Requested delay in milliseconds | -### End attributes +#### End attributes All end attributes are optional completion enrichment. @@ -270,11 +320,11 @@ All end attributes are optional completion enrichment. |---|---|---|---|---| | `pi.sleep.outcome` | `string` | elapsed, aborted | | Delay outcome | -### Events +#### Events No declared span events. -## `pi.harness.event_handler` +### `pi.harness.event_handler` One passive event listener invocation @@ -282,14 +332,14 @@ One passive event listener invocation - Default status: `ok` - Error when: The listener throws -### Start attributes +#### Start attributes | Name | Type | Required | Values | Notes | Description | |---|---|---:|---|---|---| | `pi.event.type` | `string` | yes | run_start, run_resume, run_suspend, run_abort, run_end, fault, handler_error, turn_start, turn_end, retry_scheduled, retry_start, retry_end, message_start, message_update, message_end, tool_start, tool_update, tool_end, entry_added, write_pending, queue_update, fact_update, config_update, compaction_start, compaction_end, navigation_start, navigation_end, lane_created, usage | low cardinality | Delivered harness event type | | `pi.lane.name` | `string` | no | | high cardinality | Lane name for lane-scoped events | -### End attributes +#### End attributes All end attributes are optional completion enrichment. @@ -297,11 +347,11 @@ All end attributes are optional completion enrichment. |---|---|---|---|---| | _none_ | | | | | -### Events +#### Events No declared span events. -## `pi.session.write` +### `pi.session.write` One committed session mutation @@ -309,7 +359,7 @@ One committed session mutation - Default status: `ok` - Error when: Storage rejects the mutation -### Start attributes +#### Start attributes | Name | Type | Required | Values | Notes | Description | |---|---|---:|---|---|---| @@ -318,7 +368,7 @@ One committed session mutation | `pi.session.mutation` | `string` | yes | entry, record, lane, fact | | Session mutation kind | | `pi.session.item_type` | `string` | no | | | Entry, record, lane, or fact subtype | -### End attributes +#### End attributes All end attributes are optional completion enrichment. @@ -326,6 +376,6 @@ All end attributes are optional completion enrichment. |---|---|---|---|---| | `pi.session.seq` | `number` | | | Committed session sequence when exposed | -### Events +#### Events No declared span events. diff --git a/packages/agent/package.json b/packages/agent/package.json index f950da6a9..437ec720b 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -36,6 +36,7 @@ }, "dependencies": { "@earendil-works/pi-ai": "^0.83.0", + "@earendil-works/pi-telemetry": "^0.83.0", "diff": "8.0.4", "ignore": "7.0.5", "typebox": "1.3.7", diff --git a/packages/agent/scripts/generate-telemetry-docs.ts b/packages/agent/scripts/generate-telemetry-docs.ts index c048d653c..65a32e3b7 100644 --- a/packages/agent/scripts/generate-telemetry-docs.ts +++ b/packages/agent/scripts/generate-telemetry-docs.ts @@ -1,12 +1,116 @@ -import { resolve } from "node:path"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; import { pathToFileURL } from "node:url"; -import { generateTelemetryDocs } from "../../ai/scripts/generate-telemetry-docs.ts"; -import { HARNESS_TELEMETRY_SCHEMA } from "../src/harness/telemetry.ts"; +import type { TelemetryAttributeDefinition, TelemetrySchemaDefinition } from "@earendil-works/pi-telemetry"; +import { AI_TELEMETRY_SCHEMA, HARNESS_TELEMETRY_SCHEMA } from "../src/harness/telemetry.ts"; + +function escapeCell(value: string): string { + return value.replaceAll("|", "\\|").replaceAll("\n", " "); +} + +function allowedValues(definition: TelemetryAttributeDefinition): string { + if ("values" in definition && definition.values) return definition.values.map(String).join(", "); + if ("elementValues" in definition && definition.elementValues) { + return `elements: ${definition.elementValues.map(String).join(", ")}`; + } + return ""; +} + +function attributeNotes(definition: TelemetryAttributeDefinition): string { + return [definition.cardinality ? `${definition.cardinality} cardinality` : "", definition.sensitive ? "sensitive" : ""] + .filter(Boolean) + .join(", "); +} + +function parentDescription(parent: TelemetrySchemaDefinition["spans"][string]["parents"]): string { + switch (parent.kind) { + case "any": + return "root or any caller span"; + case "root_or_external": + return "root or caller-owned external span"; + case "spans": + return parent.spans.map((span) => `\`${span}\``).join(", "); + } +} + +function renderSchema(schema: TelemetrySchemaDefinition, title: string): string[] { + const lines = [`## ${title}`, "", `Schema version: ${schema.version}`, ""]; + for (const [spanName, span] of Object.entries(schema.spans)) { + lines.push(`### \`${spanName}\``, "", span.description, ""); + lines.push(`- Parents: ${parentDescription(span.parents)}`); + lines.push(`- Default status: \`${span.status.default}\``); + lines.push(`- Error when: ${span.status.errorWhen}`, ""); + lines.push("#### Start attributes", ""); + lines.push("| Name | Type | Required | Values | Notes | Description |"); + lines.push("|---|---|---:|---|---|---|"); + for (const [name, definition] of Object.entries(span.startAttributes)) { + lines.push( + `| \`${name}\` | \`${definition.type}\` | ${definition.required ? "yes" : "no"} | ${escapeCell(allowedValues(definition))} | ${escapeCell(attributeNotes(definition))} | ${escapeCell(definition.description)} |`, + ); + } + if (Object.keys(span.startAttributes).length === 0) lines.push("| _none_ | | | | | |"); + lines.push("", "#### End attributes", ""); + lines.push("All end attributes are optional completion enrichment.", ""); + lines.push("| Name | Type | Values | Notes | Description |"); + lines.push("|---|---|---|---|---|"); + for (const [name, definition] of Object.entries(span.endAttributes)) { + lines.push( + `| \`${name}\` | \`${definition.type}\` | ${escapeCell(allowedValues(definition))} | ${escapeCell(attributeNotes(definition))} | ${escapeCell(definition.description)} |`, + ); + } + if (Object.keys(span.endAttributes).length === 0) lines.push("| _none_ | | | | |"); + lines.push("", "#### Events", ""); + const events = Object.entries(span.events ?? {}); + if (events.length === 0) { + lines.push("No declared span events.", ""); + continue; + } + for (const [eventName, event] of events) { + lines.push(`##### \`${eventName}\``, "", event.description, ""); + lines.push("| Name | Type | Required | Values | Notes | Description |"); + lines.push("|---|---|---:|---|---|---|"); + for (const [name, definition] of Object.entries(event.attributes)) { + lines.push( + `| \`${name}\` | \`${definition.type}\` | ${definition.required ? "yes" : "no"} | ${escapeCell(allowedValues(definition))} | ${escapeCell(attributeNotes(definition))} | ${escapeCell(definition.description)} |`, + ); + } + if (Object.keys(event.attributes).length === 0) lines.push("| _none_ | | | | | |"); + lines.push(""); + } + } + return lines; +} + +export function renderAgentTelemetrySchemaMarkdown(): string { + const lines = [ + "# Pi Agent Telemetry Schemas", + "", + "", + "", + ...renderSchema(AI_TELEMETRY_SCHEMA, "AI request schema"), + ...renderSchema(HARNESS_TELEMETRY_SCHEMA, "Harness schema"), + ]; + return `${lines.join("\n").trimEnd()}\n`; +} + +export function generateTelemetryDocs(outputPath: string, check: boolean): void { + const expected = renderAgentTelemetrySchemaMarkdown(); + if (check) { + let actual = ""; + try { + actual = readFileSync(outputPath, "utf8"); + } catch { + throw new Error(`${outputPath} is missing; run the telemetry documentation generator`); + } + if (actual !== expected) throw new Error(`${outputPath} is stale; run the telemetry documentation generator`); + return; + } + mkdirSync(dirname(outputPath), { recursive: true }); + writeFileSync(outputPath, expected); +} if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { generateTelemetryDocs( - HARNESS_TELEMETRY_SCHEMA, - "Pi Agent Harness Telemetry Schema", resolve(import.meta.dirname, "../docs/telemetry-schema.md"), process.argv.includes("--check"), ); diff --git a/packages/agent/src/harness/telemetry.ts b/packages/agent/src/harness/telemetry.ts index 2695c4aa2..159108279 100644 --- a/packages/agent/src/harness/telemetry.ts +++ b/packages/agent/src/harness/telemetry.ts @@ -10,7 +10,7 @@ import type { TelemetrySchemaSpanStartAttributes, TelemetrySchemaSpanUnion, TelemetrySpan, -} from "@earendil-works/pi-ai"; +} from "@earendil-works/pi-telemetry"; export type { AttributeValue, @@ -36,7 +36,112 @@ export type { TelemetrySpan, TelemetrySpanDefinition, TelemetryStartAttributeDefinition, -} from "@earendil-works/pi-ai"; +} from "@earendil-works/pi-telemetry"; + +export const AI_TELEMETRY_SCHEMA = { + version: 1, + spans: { + "pi.ai.request": { + description: "One logical request to an AI provider", + parents: { kind: "any" }, + startAttributes: { + "pi.ai.operation": { + type: "string", + required: true, + values: ["stream", "fetch_deferred", "cancel_deferred", "generate_images"], + description: "Logical provider operation", + }, + "pi.ai.provider": { + type: "string", + required: true, + description: "Selected provider id", + }, + "pi.ai.model": { + type: "string", + required: true, + description: "Requested model id", + }, + "pi.ai.api": { + type: "string", + required: true, + description: "Provider API id", + }, + "pi.ai.streaming": { + type: "boolean", + required: true, + description: "Whether this operation returns a stream", + }, + "pi.ai.deferred": { + type: "boolean", + required: false, + description: "Whether the operation requests or participates in deferred execution", + }, + }, + endAttributes: { + "pi.ai.response.model": { type: "string", description: "Concrete response model" }, + "pi.ai.response.id": { + type: "string", + cardinality: "high", + description: "Provider response id", + }, + "pi.ai.response.stop_reason": { + type: "string", + values: ["stop", "length", "tool_use", "error", "aborted", "deferred"], + description: "Normalized terminal response reason", + }, + "pi.ai.http.status_code": { type: "number", description: "Final HTTP status" }, + "pi.ai.usage.input_tokens": { type: "number", description: "Reported input tokens" }, + "pi.ai.usage.output_tokens": { type: "number", description: "Reported output tokens" }, + "pi.ai.usage.cache_read_tokens": { type: "number", description: "Reported cache-read tokens" }, + "pi.ai.usage.cache_write_tokens": { + type: "number", + description: "Reported cache-write tokens", + }, + "pi.ai.usage.reasoning_tokens": { type: "number", description: "Reported reasoning tokens" }, + "pi.ai.usage.total_tokens": { type: "number", description: "Reported total tokens" }, + "pi.ai.usage.cost": { type: "number", description: "Reported total cost" }, + "pi.ai.stream.chunk_count": { type: "number", description: "Streamed update chunk count" }, + "pi.ai.stream.time_to_first_chunk_ms": { + type: "number", + description: "Elapsed milliseconds to first update chunk", + }, + "pi.ai.error.type": { + type: "string", + cardinality: "low", + description: "Provider or transport error class", + }, + }, + status: { default: "ok", errorWhen: "The operation throws or returns an error result" }, + }, + }, +} as const satisfies TelemetrySchemaDefinition; + +export type AiSpanName = TelemetrySchemaSpanName; +export type AiSpanStartAttributes = TelemetrySchemaSpanStartAttributes< + typeof AI_TELEMETRY_SCHEMA, + Name +>; +export type AiSpanEndAttributes = TelemetrySchemaSpanEndAttributes< + typeof AI_TELEMETRY_SCHEMA, + Name +>; +export type AiSpanAttributes = AiSpanStartAttributes & AiSpanEndAttributes; +export type AiSpanEventName = TelemetrySchemaSpanEventName; +export type AiSpanEventAttributes< + Name extends AiSpanName, + EventName extends AiSpanEventName, +> = TelemetrySchemaSpanEventAttributes; +export type AiTelemetrySpan = SchemaTelemetrySpan; +export type AiSpan = TelemetrySchemaSpanUnion; + +export function startAiSpan, Result>( + telemetryContext: TelemetryContext, + name: Name, + attributes: ExactTelemetryAttributes, Attributes>, + callback: (span: AiTelemetrySpan) => Result | Promise, +): Promise { + return telemetryContext.startSpan({ name, attributes }, (span) => callback(span as AiTelemetrySpan)); +} const HOOK_NAMES = [ "before_run", diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 4882547a4..998c6ed54 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -1,14 +1,7 @@ // Core Agent +export { uuidv7 } from "@earendil-works/pi-ai"; export type { - AiSpan, - AiSpanAttributes, - AiSpanEndAttributes, - AiSpanEventAttributes, - AiSpanEventName, - AiSpanName, - AiSpanStartAttributes, - AiTelemetrySpan, AttributeValue, ExactTelemetryAttributes, InferEventAttributes, @@ -37,14 +30,8 @@ export type { TelemetrySpan, TelemetrySpanDefinition, TelemetryStartAttributeDefinition, -} from "@earendil-works/pi-ai"; -export { - AI_TELEMETRY_SCHEMA, - defineTelemetrySchema, - NOOP_TELEMETRY_CONTEXT, - startAiSpan, - uuidv7, -} from "@earendil-works/pi-ai"; +} from "@earendil-works/pi-telemetry"; +export { defineTelemetrySchema, NOOP_TELEMETRY_CONTEXT } from "@earendil-works/pi-telemetry"; export * from "./agent.ts"; // Loop functions export * from "./agent-loop.ts"; @@ -87,6 +74,14 @@ export * from "./harness/session/search.ts"; export * from "./harness/skills.ts"; export * from "./harness/system-prompt.ts"; export type { + AiSpan, + AiSpanAttributes, + AiSpanEndAttributes, + AiSpanEventAttributes, + AiSpanEventName, + AiSpanName, + AiSpanStartAttributes, + AiTelemetrySpan, HarnessSpan, HarnessSpanAttributes, HarnessSpanEndAttributes, @@ -96,7 +91,12 @@ export type { HarnessSpanStartAttributes, HarnessTelemetrySpan, } from "./harness/telemetry.ts"; -export { HARNESS_TELEMETRY_SCHEMA, startHarnessSpan } from "./harness/telemetry.ts"; +export { + AI_TELEMETRY_SCHEMA, + HARNESS_TELEMETRY_SCHEMA, + startAiSpan, + startHarnessSpan, +} from "./harness/telemetry.ts"; export * from "./harness/tools/index.ts"; export { type AgentHarnessResources, diff --git a/packages/agent/test/harness/telemetry.test.ts b/packages/agent/test/harness/telemetry.test.ts index 23c9e2e9f..e344a4350 100644 --- a/packages/agent/test/harness/telemetry.test.ts +++ b/packages/agent/test/harness/telemetry.test.ts @@ -1,17 +1,23 @@ import { readFileSync } from "node:fs"; import { resolve } from "node:path"; -import { NOOP_TELEMETRY_CONTEXT, type TelemetryContext } from "@earendil-works/pi-ai"; +import { NOOP_TELEMETRY_CONTEXT, type TelemetryContext } from "@earendil-works/pi-telemetry"; import { describe, expect, expectTypeOf, it } from "vitest"; -import { renderTelemetrySchemaMarkdown } from "../../../ai/scripts/generate-telemetry-docs.ts"; +import { renderAgentTelemetrySchemaMarkdown } from "../../scripts/generate-telemetry-docs.ts"; import { + AI_TELEMETRY_SCHEMA, + type AiSpanEndAttributes, + type AiSpanStartAttributes, HARNESS_TELEMETRY_SCHEMA, type HarnessSpanEndAttributes, type HarnessSpanStartAttributes, + startAiSpan, startHarnessSpan, } from "../../src/harness/telemetry.ts"; -describe("HARNESS_TELEMETRY_SCHEMA", () => { - it("is serializable, exhaustive, and generates the checked-in reference", () => { +describe("agent telemetry schemas", () => { + it("serializes both schemas and generates the checked-in reference", () => { + expect(() => JSON.stringify(AI_TELEMETRY_SCHEMA)).not.toThrow(); + expect(() => JSON.stringify(HARNESS_TELEMETRY_SCHEMA)).not.toThrow(); expect(Object.keys(HARNESS_TELEMETRY_SCHEMA.spans)).toEqual([ "pi.harness.run", "pi.harness.compaction", @@ -25,13 +31,61 @@ describe("HARNESS_TELEMETRY_SCHEMA", () => { "pi.harness.event_handler", "pi.session.write", ]); - expect(() => JSON.stringify(HARNESS_TELEMETRY_SCHEMA)).not.toThrow(); - const expected = renderTelemetrySchemaMarkdown(HARNESS_TELEMETRY_SCHEMA, "Pi Agent Harness Telemetry Schema"); const actual = readFileSync(resolve(import.meta.dirname, "../../docs/telemetry-schema.md"), "utf8"); - expect(actual).toBe(expected); + expect(actual).toBe(renderAgentTelemetrySchemaMarkdown()); }); - it("infers per-span literals and optional completion enrichment", async () => { + it("infers exact AI start and optional end attributes", async () => { + type Start = AiSpanStartAttributes<"pi.ai.request">; + type End = AiSpanEndAttributes<"pi.ai.request">; + expectTypeOf().toMatchTypeOf<{ + "pi.ai.operation": "stream" | "fetch_deferred" | "cancel_deferred" | "generate_images"; + "pi.ai.provider": string; + "pi.ai.model": string; + "pi.ai.api": string; + "pi.ai.streaming": boolean; + "pi.ai.deferred"?: boolean; + }>(); + expectTypeOf().toEqualTypeOf< + "stop" | "length" | "tool_use" | "error" | "aborted" | "deferred" | undefined + >(); + + const telemetryContext: TelemetryContext = NOOP_TELEMETRY_CONTEXT; + await startAiSpan( + telemetryContext, + "pi.ai.request", + { + "pi.ai.operation": "stream", + "pi.ai.provider": "provider", + "pi.ai.model": "model", + "pi.ai.api": "api", + "pi.ai.streaming": true, + }, + (span) => { + span.setAttributes({ "pi.ai.response.stop_reason": "tool_use" }); + // @ts-expect-error pi.ai.request declares no span events + span.addEvent("chunk"); + }, + ); + + const compileTimeFailures = () => { + const extraAttributes = { + "pi.ai.operation": "stream", + "pi.ai.provider": "provider", + "pi.ai.model": "model", + "pi.ai.api": "api", + "pi.ai.streaming": true, + "pi.ai.unknown": true, + } as const; + // @ts-expect-error variables with unknown attributes are rejected + void startAiSpan(telemetryContext, "pi.ai.request", extraAttributes, () => {}); + // @ts-expect-error missing required start attributes + void startAiSpan(telemetryContext, "pi.ai.request", { "pi.ai.operation": "stream" }, () => {}); + }; + expectTypeOf(compileTimeFailures).toBeFunction(); + }); + + it("infers per-span harness literals and optional completion enrichment", async () => { type RunStart = HarnessSpanStartAttributes<"pi.harness.run">; type RunEnd = HarnessSpanEndAttributes<"pi.harness.run">; expectTypeOf().toEqualTypeOf<"run">(); diff --git a/packages/agent/tsconfig.build.json b/packages/agent/tsconfig.build.json index 979dbc899..7761b01a1 100644 --- a/packages/agent/tsconfig.build.json +++ b/packages/agent/tsconfig.build.json @@ -3,6 +3,7 @@ "compilerOptions": { "outDir": "./dist", "paths": { + "@earendil-works/pi-telemetry": ["../telemetry/dist/index.d.ts"], "@earendil-works/pi-ai": ["../ai/dist/index.d.ts"], "@earendil-works/pi-ai/*": ["../ai/dist/*.d.ts", "../ai/dist/providers/*.d.ts"] }, diff --git a/packages/agent/vitest.config.ts b/packages/agent/vitest.config.ts index ed0ab064a..39600da16 100644 --- a/packages/agent/vitest.config.ts +++ b/packages/agent/vitest.config.ts @@ -1,6 +1,7 @@ import { fileURLToPath } from "node:url"; import { defineConfig } from "vitest/config"; +const telemetrySrcIndex = fileURLToPath(new URL("../telemetry/src/index.ts", import.meta.url)); const aiSrcIndex = fileURLToPath(new URL("../ai/src/index.ts", import.meta.url)); const aiSrcCompat = fileURLToPath(new URL("../ai/src/compat.ts", import.meta.url)); const agentSrcIndex = fileURLToPath(new URL("./src/index.ts", import.meta.url)); @@ -15,6 +16,7 @@ export default defineConfig({ }, resolve: { alias: [ + { find: /^@earendil-works\/pi-telemetry$/, replacement: telemetrySrcIndex }, { find: /^@earendil-works\/pi-agent-core$/, replacement: agentSrcIndex }, { find: /^@earendil-works\/pi-ai$/, replacement: aiSrcIndex }, { find: /^@earendil-works\/pi-ai\/compat$/, replacement: aiSrcCompat }, diff --git a/packages/agent/vitest.harness.config.ts b/packages/agent/vitest.harness.config.ts index 8045bb1a6..964c59a48 100644 --- a/packages/agent/vitest.harness.config.ts +++ b/packages/agent/vitest.harness.config.ts @@ -1,6 +1,7 @@ import { fileURLToPath } from "node:url"; import { defineConfig } from "vitest/config"; +const telemetrySrcIndex = fileURLToPath(new URL("../telemetry/src/index.ts", import.meta.url)); const aiSrcIndex = fileURLToPath(new URL("../ai/src/index.ts", import.meta.url)); const aiSrcCompat = fileURLToPath(new URL("../ai/src/compat.ts", import.meta.url)); const agentSrcIndex = fileURLToPath(new URL("../agent/src/index.ts", import.meta.url)); @@ -22,6 +23,7 @@ export default defineConfig({ }, resolve: { alias: [ + { find: /^@earendil-works\/pi-telemetry$/, replacement: telemetrySrcIndex }, { find: /^@earendil-works\/pi-agent-core$/, replacement: agentSrcIndex }, { find: /^@earendil-works\/pi-ai$/, replacement: aiSrcIndex }, { find: /^@earendil-works\/pi-ai\/compat$/, replacement: aiSrcCompat }, diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 31789f037..e967138d0 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -68,7 +68,7 @@ ### Added -- Added the callback-based telemetry context contract, typed pi-ai telemetry schema, generated schema reference, and explicit telemetry context propagation across stream, deferred, and image request options. +- Added explicit `TelemetryContext` propagation across stream, deferred, and image request options using the vendor-neutral `@earendil-works/pi-telemetry` contract. - Added deferred provider request contracts, durable response handles, authenticated fetch/cancel dispatch, and faux-provider support for pending, ready, failed, and cancelled responses ([#7339](https://github.com/earendil-works/pi/pull/7339) by [@davidbrai](https://github.com/davidbrai)). - Added Baseten as a built-in OpenAI-compatible provider with models.dev catalog generation and native `chat_template_args` reasoning controls. - Added arbitrary OpenAI-compatible sampling parameters through `Model.samplingParams` and `StreamOptions.samplingParams`, including per-request overrides ([#7568](https://github.com/earendil-works/pi/pull/7568) by [@mrexodia](https://github.com/mrexodia)). diff --git a/packages/ai/docs/telemetry-schema.md b/packages/ai/docs/telemetry-schema.md deleted file mode 100644 index 9d582e991..000000000 --- a/packages/ai/docs/telemetry-schema.md +++ /dev/null @@ -1,49 +0,0 @@ -# Pi AI Telemetry Schema - - - -Schema version: 1 - -## `pi.ai.request` - -One logical request to an AI provider - -- Parents: root or any caller span -- Default status: `ok` -- Error when: The operation throws or returns an error result - -### Start attributes - -| Name | Type | Required | Values | Notes | Description | -|---|---|---:|---|---|---| -| `pi.ai.operation` | `string` | yes | stream, fetch_deferred, cancel_deferred, generate_images | | Logical provider operation | -| `pi.ai.provider` | `string` | yes | | | Selected provider id | -| `pi.ai.model` | `string` | yes | | | Requested model id | -| `pi.ai.api` | `string` | yes | | | Provider API id | -| `pi.ai.streaming` | `boolean` | yes | | | Whether this operation returns a stream | -| `pi.ai.deferred` | `boolean` | no | | | Whether the operation requests or participates in deferred execution | - -### End attributes - -All end attributes are optional completion enrichment. - -| Name | Type | Values | Notes | Description | -|---|---|---|---|---| -| `pi.ai.response.model` | `string` | | | Concrete response model | -| `pi.ai.response.id` | `string` | | high cardinality | Provider response id | -| `pi.ai.response.stop_reason` | `string` | stop, length, tool_use, error, aborted, deferred | | Normalized terminal response reason | -| `pi.ai.http.status_code` | `number` | | | Final HTTP status | -| `pi.ai.usage.input_tokens` | `number` | | | Reported input tokens | -| `pi.ai.usage.output_tokens` | `number` | | | Reported output tokens | -| `pi.ai.usage.cache_read_tokens` | `number` | | | Reported cache-read tokens | -| `pi.ai.usage.cache_write_tokens` | `number` | | | Reported cache-write tokens | -| `pi.ai.usage.reasoning_tokens` | `number` | | | Reported reasoning tokens | -| `pi.ai.usage.total_tokens` | `number` | | | Reported total tokens | -| `pi.ai.usage.cost` | `number` | | | Reported total cost | -| `pi.ai.stream.chunk_count` | `number` | | | Streamed update chunk count | -| `pi.ai.stream.time_to_first_chunk_ms` | `number` | | | Elapsed milliseconds to first update chunk | -| `pi.ai.error.type` | `string` | | low cardinality | Provider or transport error class | - -### Events - -No declared span events. diff --git a/packages/ai/package.json b/packages/ai/package.json index cb38d2051..8d9c47a24 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -53,8 +53,6 @@ "hydrate-model-data": "node scripts/generate-models.ts --strict --data-only", "generate-model-catalog": "node scripts/generate-models.ts --strict --json-only --json-output ../../.artifacts/model-catalog", "generate-image-models": "node scripts/generate-image-models.ts --strict", - "generate-telemetry-docs": "node scripts/generate-telemetry-docs.ts", - "check:telemetry-docs": "node scripts/generate-telemetry-docs.ts --check", "check:model-data": "node scripts/check-model-data.ts", "build": "npm run generate-models && npm run build:offline", "build:offline": "npm run check:model-data && tsgo -p tsconfig.build.json && shx rm -rf dist/providers/data && shx cp -r src/providers/data dist/providers/data", @@ -64,6 +62,7 @@ "dependencies": { "@anthropic-ai/sdk": "0.91.1", "@aws-sdk/client-bedrock-runtime": "3.1048.0", + "@earendil-works/pi-telemetry": "^0.83.0", "@google/genai": "1.52.0", "@mistralai/mistralai": "2.2.6", "@opentelemetry/api": "1.9.0", diff --git a/packages/ai/scripts/generate-telemetry-docs.ts b/packages/ai/scripts/generate-telemetry-docs.ts deleted file mode 100644 index decd96250..000000000 --- a/packages/ai/scripts/generate-telemetry-docs.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { dirname, resolve } from "node:path"; -import { pathToFileURL } from "node:url"; -import { AI_TELEMETRY_SCHEMA, type TelemetryAttributeDefinition, type TelemetrySchemaDefinition } from "../src/telemetry.ts"; - -function escapeCell(value: string): string { - return value.replaceAll("|", "\\|").replaceAll("\n", " "); -} - -function allowedValues(definition: TelemetryAttributeDefinition): string { - if ("values" in definition && definition.values) return definition.values.map(String).join(", "); - if ("elementValues" in definition && definition.elementValues) { - return `elements: ${definition.elementValues.map(String).join(", ")}`; - } - return ""; -} - -function attributeNotes(definition: TelemetryAttributeDefinition): string { - return [definition.cardinality ? `${definition.cardinality} cardinality` : "", definition.sensitive ? "sensitive" : ""] - .filter(Boolean) - .join(", "); -} - -function parentDescription(parent: TelemetrySchemaDefinition["spans"][string]["parents"]): string { - switch (parent.kind) { - case "any": - return "root or any caller span"; - case "root_or_external": - return "root or caller-owned external span"; - case "spans": - return parent.spans.map((span) => `\`${span}\``).join(", "); - } -} - -export function renderTelemetrySchemaMarkdown(schema: TelemetrySchemaDefinition, title: string): string { - const lines = [ - `# ${title}`, - "", - "", - "", - `Schema version: ${schema.version}`, - "", - ]; - - for (const [spanName, span] of Object.entries(schema.spans)) { - lines.push(`## \`${spanName}\``, "", span.description, ""); - lines.push(`- Parents: ${parentDescription(span.parents)}`); - lines.push(`- Default status: \`${span.status.default}\``); - lines.push(`- Error when: ${span.status.errorWhen}`, ""); - lines.push("### Start attributes", ""); - lines.push("| Name | Type | Required | Values | Notes | Description |"); - lines.push("|---|---|---:|---|---|---|"); - for (const [name, definition] of Object.entries(span.startAttributes)) { - lines.push( - `| \`${name}\` | \`${definition.type}\` | ${definition.required ? "yes" : "no"} | ${escapeCell(allowedValues(definition))} | ${escapeCell(attributeNotes(definition))} | ${escapeCell(definition.description)} |`, - ); - } - if (Object.keys(span.startAttributes).length === 0) lines.push("| _none_ | | | | | |"); - lines.push("", "### End attributes", ""); - lines.push("All end attributes are optional completion enrichment.", ""); - lines.push("| Name | Type | Values | Notes | Description |"); - lines.push("|---|---|---|---|---|"); - for (const [name, definition] of Object.entries(span.endAttributes)) { - lines.push( - `| \`${name}\` | \`${definition.type}\` | ${escapeCell(allowedValues(definition))} | ${escapeCell(attributeNotes(definition))} | ${escapeCell(definition.description)} |`, - ); - } - if (Object.keys(span.endAttributes).length === 0) lines.push("| _none_ | | | | |"); - lines.push("", "### Events", ""); - const events = Object.entries(span.events ?? {}); - if (events.length === 0) { - lines.push("No declared span events.", ""); - continue; - } - for (const [eventName, event] of events) { - lines.push(`#### \`${eventName}\``, "", event.description, ""); - lines.push("| Name | Type | Required | Values | Notes | Description |"); - lines.push("|---|---|---:|---|---|---|"); - for (const [name, definition] of Object.entries(event.attributes)) { - lines.push( - `| \`${name}\` | \`${definition.type}\` | ${definition.required ? "yes" : "no"} | ${escapeCell(allowedValues(definition))} | ${escapeCell(attributeNotes(definition))} | ${escapeCell(definition.description)} |`, - ); - } - if (Object.keys(event.attributes).length === 0) lines.push("| _none_ | | | | | |"); - lines.push(""); - } - } - - return `${lines.join("\n").trimEnd()}\n`; -} - -export function generateTelemetryDocs( - schema: TelemetrySchemaDefinition, - title: string, - outputPath: string, - check: boolean, -): void { - const expected = renderTelemetrySchemaMarkdown(schema, title); - if (check) { - let actual = ""; - try { - actual = readFileSync(outputPath, "utf8"); - } catch { - throw new Error(`${outputPath} is missing; run the telemetry documentation generator`); - } - if (actual !== expected) throw new Error(`${outputPath} is stale; run the telemetry documentation generator`); - return; - } - mkdirSync(dirname(outputPath), { recursive: true }); - writeFileSync(outputPath, expected); -} - -if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { - generateTelemetryDocs( - AI_TELEMETRY_SCHEMA, - "Pi AI Telemetry Schema", - resolve(import.meta.dirname, "../docs/telemetry-schema.md"), - process.argv.includes("--check"), - ); -} diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 63d507400..4ff678102 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -35,7 +35,6 @@ export * from "./models.ts"; export * from "./models-store.ts"; export * from "./providers/faux.ts"; export * from "./session-resources.ts"; -export * from "./telemetry.ts"; export * from "./types.ts"; export * from "./utils/diagnostics.ts"; export * from "./utils/event-stream.ts"; diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 82a3a9e1e..1cdacdbae 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -1,3 +1,4 @@ +import type { TelemetryContext } from "@earendil-works/pi-telemetry"; import type { AnthropicOptions } from "./api/anthropic-messages.ts"; import type { AzureOpenAIResponsesOptions } from "./api/azure-openai-responses.ts"; import type { BedrockOptions } from "./api/bedrock-converse-stream.ts"; @@ -8,7 +9,6 @@ import type { OpenAICodexResponsesOptions } from "./api/openai-codex-responses.t import type { OpenAICompletionsOptions } from "./api/openai-completions.ts"; import type { OpenAIResponsesOptions } from "./api/openai-responses.ts"; import type { PiMessagesOptions } from "./api/pi-messages.ts"; -import type { TelemetryContext } from "./telemetry.ts"; import type { AssistantMessageDiagnostic } from "./utils/diagnostics.ts"; import type { AssistantMessageEventStream } from "./utils/event-stream.ts"; diff --git a/packages/ai/test/telemetry-options.test.ts b/packages/ai/test/telemetry-options.test.ts index 732332db8..fc4926369 100644 --- a/packages/ai/test/telemetry-options.test.ts +++ b/packages/ai/test/telemetry-options.test.ts @@ -1,10 +1,10 @@ +import { NOOP_TELEMETRY_CONTEXT, type TelemetryContext } from "@earendil-works/pi-telemetry"; import { describe, expect, it } from "vitest"; import { buildBaseOptions } from "../src/api/simple-options.ts"; import { generateImages } from "../src/images.ts"; import { registerImagesApiProvider } from "../src/images-api-registry.ts"; import { createImagesModels, createImagesProvider } from "../src/images-models.ts"; import { createModels, createProvider } from "../src/models.ts"; -import { NOOP_TELEMETRY_CONTEXT, type TelemetryContext } from "../src/telemetry.ts"; import type { Context, DeferredHandle, diff --git a/packages/ai/test/telemetry.test.ts b/packages/ai/test/telemetry.test.ts deleted file mode 100644 index 8e037a663..000000000 --- a/packages/ai/test/telemetry.test.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { readFileSync } from "node:fs"; -import { resolve } from "node:path"; -import { describe, expect, expectTypeOf, it } from "vitest"; -import { renderTelemetrySchemaMarkdown } from "../scripts/generate-telemetry-docs.ts"; -import { - AI_TELEMETRY_SCHEMA, - type AiSpanEndAttributes, - type AiSpanStartAttributes, - NOOP_TELEMETRY_CONTEXT, - startAiSpan, - type TelemetryContext, - type TelemetrySpan, -} from "../src/telemetry.ts"; - -function unreadable(value: T): T { - return new Proxy(value, { - get: () => { - throw new Error("read"); - }, - getOwnPropertyDescriptor: () => { - throw new Error("inspect"); - }, - ownKeys: () => { - throw new Error("enumerate"); - }, - }); -} - -describe("NOOP_TELEMETRY_CONTEXT", () => { - it("admits callbacks synchronously and reuses one inert span", async () => { - let admitted = false; - let firstSpan: TelemetrySpan | undefined; - const result = NOOP_TELEMETRY_CONTEXT.startSpan({ name: "first" }, async (span) => { - admitted = true; - firstSpan = span; - const child = await span.startSpan({ name: "child" }, (childSpan) => childSpan); - expect(child).toBe(span); - return 42; - }); - - expect(admitted).toBe(true); - expect(await result).toBe(42); - expect(Object.isFrozen(firstSpan)).toBe(true); - }); - - it("preserves synchronous and asynchronous rejection values", async () => { - const syncError = { kind: "sync" }; - const sync = NOOP_TELEMETRY_CONTEXT.startSpan({ name: "sync" }, () => { - throw syncError; - }); - await expect(sync).rejects.toBe(syncError); - - const asyncError = { kind: "async" }; - const asyncResult = NOOP_TELEMETRY_CONTEXT.startSpan({ name: "async" }, async () => { - throw asyncError; - }); - await expect(asyncResult).rejects.toBe(asyncError); - }); - - it("does not inspect or retain telemetry payloads", async () => { - const options = unreadable({ name: "operation", attributes: { secret: "prompt content" } }); - await NOOP_TELEMETRY_CONTEXT.startSpan(options, (span) => { - expect(() => span.addEvent("event", unreadable({ secret: "content" }))).not.toThrow(); - expect(() => span.setAttributes(unreadable({ secret: "content" }))).not.toThrow(); - expect(() => span.setStatus(unreadable({ status: "ok" as const }))).not.toThrow(); - }); - }); -}); - -describe("AI_TELEMETRY_SCHEMA", () => { - it("is serializable and generates the checked-in reference", () => { - expect(() => JSON.stringify(AI_TELEMETRY_SCHEMA)).not.toThrow(); - const expected = renderTelemetrySchemaMarkdown(AI_TELEMETRY_SCHEMA, "Pi AI Telemetry Schema"); - const actual = readFileSync(resolve(import.meta.dirname, "../docs/telemetry-schema.md"), "utf8"); - expect(actual).toBe(expected); - }); - - it("infers exact start and optional end attributes", async () => { - type Start = AiSpanStartAttributes<"pi.ai.request">; - type End = AiSpanEndAttributes<"pi.ai.request">; - expectTypeOf().toMatchTypeOf<{ - "pi.ai.operation": "stream" | "fetch_deferred" | "cancel_deferred" | "generate_images"; - "pi.ai.provider": string; - "pi.ai.model": string; - "pi.ai.api": string; - "pi.ai.streaming": boolean; - "pi.ai.deferred"?: boolean; - }>(); - expectTypeOf().toEqualTypeOf< - "stop" | "length" | "tool_use" | "error" | "aborted" | "deferred" | undefined - >(); - - const context: TelemetryContext = NOOP_TELEMETRY_CONTEXT; - await startAiSpan( - context, - "pi.ai.request", - { - "pi.ai.operation": "stream", - "pi.ai.provider": "provider", - "pi.ai.model": "model", - "pi.ai.api": "api", - "pi.ai.streaming": true, - }, - (span) => { - span.setAttributes({ "pi.ai.response.stop_reason": "tool_use" }); - // @ts-expect-error pi.ai.request declares no span events - span.addEvent("chunk"); - }, - ); - - const compileTimeFailures = () => { - const extraAttributes = { - "pi.ai.operation": "stream", - "pi.ai.provider": "provider", - "pi.ai.model": "model", - "pi.ai.api": "api", - "pi.ai.streaming": true, - "pi.ai.unknown": true, - } as const; - // @ts-expect-error variables with unknown attributes are rejected - void startAiSpan(context, "pi.ai.request", extraAttributes, () => {}); - // @ts-expect-error missing required start attributes - void startAiSpan(context, "pi.ai.request", { "pi.ai.operation": "stream" }, () => {}); - void startAiSpan( - context, - "pi.ai.request", - { - "pi.ai.operation": "stream", - "pi.ai.provider": "provider", - "pi.ai.model": "model", - "pi.ai.api": "api", - "pi.ai.streaming": true, - // @ts-expect-error unknown start attribute - "pi.ai.unknown": true, - }, - () => {}, - ); - void startAiSpan( - context, - "pi.ai.request", - { - // @ts-expect-error invalid closed-set value - "pi.ai.operation": "invalid", - "pi.ai.provider": "provider", - "pi.ai.model": "model", - "pi.ai.api": "api", - "pi.ai.streaming": true, - }, - () => {}, - ); - }; - expectTypeOf(compileTimeFailures).toBeFunction(); - }); -}); diff --git a/packages/ai/vitest.config.ts b/packages/ai/vitest.config.ts index f82ab91b7..84c74692f 100644 --- a/packages/ai/vitest.config.ts +++ b/packages/ai/vitest.config.ts @@ -1,11 +1,17 @@ -import { defineConfig } from 'vitest/config'; +import { fileURLToPath } from "node:url"; +import { defineConfig } from "vitest/config"; + +const telemetrySrcIndex = fileURLToPath(new URL("../telemetry/src/index.ts", import.meta.url)); export default defineConfig({ - test: { - globals: true, - environment: 'node', - testTimeout: 30000, // 30 seconds for API calls - reporters: process.env.GITHUB_ACTIONS ? ['dot', 'github-actions'] : ['dot'], - silent: 'passed-only', - } + test: { + globals: true, + environment: "node", + testTimeout: 30000, // 30 seconds for API calls + reporters: process.env.GITHUB_ACTIONS ? ["dot", "github-actions"] : ["dot"], + silent: "passed-only", + }, + resolve: { + alias: [{ find: /^@earendil-works\/pi-telemetry$/, replacement: telemetrySrcIndex }], + }, }); \ No newline at end of file diff --git a/packages/coding-agent/install-lock/package-lock.json b/packages/coding-agent/install-lock/package-lock.json index 7dc318ce4..89af537ae 100644 --- a/packages/coding-agent/install-lock/package-lock.json +++ b/packages/coding-agent/install-lock/package-lock.json @@ -455,6 +455,7 @@ "license": "MIT", "dependencies": { "@earendil-works/pi-ai": "^0.83.0", + "@earendil-works/pi-telemetry": "^0.83.0", "diff": "8.0.4", "ignore": "7.0.5", "typebox": "1.3.7", @@ -471,6 +472,7 @@ "dependencies": { "@anthropic-ai/sdk": "0.91.1", "@aws-sdk/client-bedrock-runtime": "3.1048.0", + "@earendil-works/pi-telemetry": "^0.83.0", "@google/genai": "1.52.0", "@mistralai/mistralai": "2.2.6", "@opentelemetry/api": "1.9.0", @@ -547,6 +549,14 @@ "node": ">=22.19.0" } }, + "node_modules/@earendil-works/pi-telemetry": { + "version": "0.83.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-telemetry/-/pi-telemetry-0.83.0.tgz", + "license": "MIT", + "engines": { + "node": ">=22.19.0" + } + }, "node_modules/@earendil-works/pi-tui": { "version": "0.83.0", "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.83.0.tgz", diff --git a/packages/coding-agent/npm-shrinkwrap.json b/packages/coding-agent/npm-shrinkwrap.json index c0cbba3c5..09092aeba 100644 --- a/packages/coding-agent/npm-shrinkwrap.json +++ b/packages/coding-agent/npm-shrinkwrap.json @@ -482,6 +482,7 @@ "license": "MIT", "dependencies": { "@earendil-works/pi-ai": "^0.83.0", + "@earendil-works/pi-telemetry": "^0.83.0", "diff": "8.0.4", "ignore": "7.0.5", "typebox": "1.3.7", @@ -498,6 +499,7 @@ "dependencies": { "@anthropic-ai/sdk": "0.91.1", "@aws-sdk/client-bedrock-runtime": "3.1048.0", + "@earendil-works/pi-telemetry": "^0.83.0", "@google/genai": "1.52.0", "@mistralai/mistralai": "2.2.6", "@opentelemetry/api": "1.9.0", @@ -537,6 +539,14 @@ "node": ">=22.19.0" } }, + "node_modules/@earendil-works/pi-telemetry": { + "version": "0.83.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-telemetry/-/pi-telemetry-0.83.0.tgz", + "license": "MIT", + "engines": { + "node": ">=22.19.0" + } + }, "node_modules/@earendil-works/pi-tui": { "version": "0.83.0", "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-0.83.0.tgz", diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 4d1bbedae..a9d256aaf 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -35,7 +35,7 @@ "scripts": { "clean": "shx rm -rf dist", "build": "tsgo -p tsconfig.build.json && shx chmod +x dist/cli.js dist/rpc-entry.js && npm run copy-assets", - "build:binary": "npm --prefix ../tui run build && npm --prefix ../ai run build && npm --prefix ../agent run build && npm --prefix ../protocol run build && npm --prefix ../client run build && npm run build && bun build --compile ./dist/bun/cli.js ./src/utils/image-resize-worker.ts --outfile dist/pi && npm run copy-binary-assets", + "build:binary": "npm --prefix ../tui run build && npm --prefix ../telemetry run build && npm --prefix ../ai run build && npm --prefix ../agent run build && npm --prefix ../protocol run build && npm --prefix ../client run build && npm run build && bun build --compile ./dist/bun/cli.js ./src/utils/image-resize-worker.ts --outfile dist/pi && npm run copy-binary-assets", "copy-assets": "shx mkdir -p dist/modes/interactive/theme && shx cp src/modes/interactive/theme/*.json dist/modes/interactive/theme/ && shx mkdir -p dist/modes/interactive/assets && shx cp src/modes/interactive/assets/*.png dist/modes/interactive/assets/ && shx mkdir -p dist/core/export-html/vendor && shx cp src/core/export-html/template.html src/core/export-html/template.css src/core/export-html/template.js dist/core/export-html/ && shx cp src/core/export-html/vendor/*.js dist/core/export-html/vendor/", "copy-binary-assets": "shx cp package.json dist/ && shx cp README.md dist/ && shx cp CHANGELOG.md dist/ && shx mkdir -p dist/theme && shx cp src/modes/interactive/theme/*.json dist/theme/ && shx mkdir -p dist/assets && shx cp src/modes/interactive/assets/*.png dist/assets/ && shx mkdir -p dist/export-html/vendor && shx cp src/core/export-html/template.html dist/export-html/ && shx cp src/core/export-html/vendor/*.js dist/export-html/vendor/ && shx cp -r docs dist/ && shx cp -r examples dist/ && shx cp ../../node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm dist/", "test": "vitest --run", diff --git a/packages/session-backends/sqlite-node/vitest.config.ts b/packages/session-backends/sqlite-node/vitest.config.ts index 7125e3274..79b5636f1 100644 --- a/packages/session-backends/sqlite-node/vitest.config.ts +++ b/packages/session-backends/sqlite-node/vitest.config.ts @@ -1,6 +1,7 @@ import { fileURLToPath } from "node:url"; import { defineConfig } from "vitest/config"; +const telemetryIndex = fileURLToPath(new URL("../../telemetry/src/index.ts", import.meta.url)); const aiIndex = fileURLToPath(new URL("../../ai/src/index.ts", import.meta.url)); const agentIndex = fileURLToPath(new URL("../../agent/src/index.ts", import.meta.url)); const agentNode = fileURLToPath(new URL("../../agent/src/node.ts", import.meta.url)); @@ -22,6 +23,7 @@ export default defineConfig({ }, resolve: { alias: [ + { find: /^@earendil-works\/pi-telemetry$/, replacement: telemetryIndex }, { find: /^@earendil-works\/pi-agent-core\/session\/testing$/, replacement: agentSessionTesting }, { find: /^@earendil-works\/pi-agent-core\/node$/, replacement: agentNode }, { find: /^@earendil-works\/pi-agent-core$/, replacement: agentIndex }, diff --git a/packages/telemetry/CHANGELOG.md b/packages/telemetry/CHANGELOG.md new file mode 100644 index 000000000..dc2f991ce --- /dev/null +++ b/packages/telemetry/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +## [Unreleased] + +### Added + +- Added the callback-based telemetry context contract, shared no-op context, and typed serializable schema utilities. diff --git a/packages/telemetry/README.md b/packages/telemetry/README.md new file mode 100644 index 000000000..02805a7a5 --- /dev/null +++ b/packages/telemetry/README.md @@ -0,0 +1,5 @@ +# @earendil-works/pi-telemetry + +Vendor-neutral telemetry contracts and typed schema utilities for pi packages. + +This package contains no exporter and depends on no telemetry backend. Applications provide a `TelemetryContext` adapter; pi packages pass contexts explicitly and define their own domain schemas. diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json new file mode 100644 index 000000000..104acd027 --- /dev/null +++ b/packages/telemetry/package.json @@ -0,0 +1,43 @@ +{ + "name": "@earendil-works/pi-telemetry", + "version": "0.83.0", + "description": "Vendor-neutral telemetry contracts and typed schema utilities for pi", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "files": [ + "dist", + "README.md" + ], + "scripts": { + "clean": "shx rm -rf dist", + "build": "tsgo -p tsconfig.build.json", + "test": "vitest --run", + "prepublishOnly": "npm run clean && npm run build" + }, + "keywords": [ + "telemetry", + "tracing", + "observability" + ], + "author": "Mario Zechner", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/earendil-works/pi.git", + "directory": "packages/telemetry" + }, + "engines": { + "node": ">=22.19.0" + }, + "devDependencies": { + "@types/node": "24.12.4", + "vitest": "4.1.9" + } +} diff --git a/packages/ai/src/telemetry.ts b/packages/telemetry/src/index.ts similarity index 71% rename from packages/ai/src/telemetry.ts rename to packages/telemetry/src/index.ts index 6bd4852e9..094812526 100644 --- a/packages/ai/src/telemetry.ts +++ b/packages/telemetry/src/index.ts @@ -268,108 +268,3 @@ export type TelemetrySchemaSpanUnion = }; }; }[TelemetrySchemaSpanName]; - -export const AI_TELEMETRY_SCHEMA = defineTelemetrySchema({ - version: 1, - spans: { - "pi.ai.request": { - description: "One logical request to an AI provider", - parents: { kind: "any" }, - startAttributes: { - "pi.ai.operation": { - type: "string", - required: true, - values: ["stream", "fetch_deferred", "cancel_deferred", "generate_images"], - description: "Logical provider operation", - }, - "pi.ai.provider": { - type: "string", - required: true, - description: "Selected provider id", - }, - "pi.ai.model": { - type: "string", - required: true, - description: "Requested model id", - }, - "pi.ai.api": { - type: "string", - required: true, - description: "Provider API id", - }, - "pi.ai.streaming": { - type: "boolean", - required: true, - description: "Whether this operation returns a stream", - }, - "pi.ai.deferred": { - type: "boolean", - required: false, - description: "Whether the operation requests or participates in deferred execution", - }, - }, - endAttributes: { - "pi.ai.response.model": { type: "string", description: "Concrete response model" }, - "pi.ai.response.id": { - type: "string", - cardinality: "high", - description: "Provider response id", - }, - "pi.ai.response.stop_reason": { - type: "string", - values: ["stop", "length", "tool_use", "error", "aborted", "deferred"], - description: "Normalized terminal response reason", - }, - "pi.ai.http.status_code": { type: "number", description: "Final HTTP status" }, - "pi.ai.usage.input_tokens": { type: "number", description: "Reported input tokens" }, - "pi.ai.usage.output_tokens": { type: "number", description: "Reported output tokens" }, - "pi.ai.usage.cache_read_tokens": { type: "number", description: "Reported cache-read tokens" }, - "pi.ai.usage.cache_write_tokens": { - type: "number", - description: "Reported cache-write tokens", - }, - "pi.ai.usage.reasoning_tokens": { type: "number", description: "Reported reasoning tokens" }, - "pi.ai.usage.total_tokens": { type: "number", description: "Reported total tokens" }, - "pi.ai.usage.cost": { type: "number", description: "Reported total cost" }, - "pi.ai.stream.chunk_count": { type: "number", description: "Streamed update chunk count" }, - "pi.ai.stream.time_to_first_chunk_ms": { - type: "number", - description: "Elapsed milliseconds to first update chunk", - }, - "pi.ai.error.type": { - type: "string", - cardinality: "low", - description: "Provider or transport error class", - }, - }, - status: { default: "ok", errorWhen: "The operation throws or returns an error result" }, - }, - }, -} as const); - -export type AiSpanName = TelemetrySchemaSpanName; -export type AiSpanStartAttributes = TelemetrySchemaSpanStartAttributes< - typeof AI_TELEMETRY_SCHEMA, - Name ->; -export type AiSpanEndAttributes = TelemetrySchemaSpanEndAttributes< - typeof AI_TELEMETRY_SCHEMA, - Name ->; -export type AiSpanAttributes = AiSpanStartAttributes & AiSpanEndAttributes; -export type AiSpanEventName = TelemetrySchemaSpanEventName; -export type AiSpanEventAttributes< - Name extends AiSpanName, - EventName extends AiSpanEventName, -> = TelemetrySchemaSpanEventAttributes; -export type AiTelemetrySpan = SchemaTelemetrySpan; -export type AiSpan = TelemetrySchemaSpanUnion; - -export function startAiSpan, Result>( - telemetryContext: TelemetryContext, - name: Name, - attributes: ExactTelemetryAttributes, Attributes>, - callback: (span: AiTelemetrySpan) => Result | Promise, -): Promise { - return telemetryContext.startSpan({ name, attributes }, (span) => callback(span as AiTelemetrySpan)); -} diff --git a/packages/telemetry/test/telemetry.test.ts b/packages/telemetry/test/telemetry.test.ts new file mode 100644 index 000000000..c09784ee0 --- /dev/null +++ b/packages/telemetry/test/telemetry.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import { + defineTelemetrySchema, + NOOP_TELEMETRY_CONTEXT, + type SchemaTelemetrySpan, + type SpanAttributes, + type SpanOptions, + type SpanStatus, + type TelemetrySchemaSpanStartAttributes, + type TelemetrySpan, +} from "../src/index.ts"; + +function unreadable(value: T): T { + return new Proxy(value, { + get: () => { + throw new Error("read"); + }, + getOwnPropertyDescriptor: () => { + throw new Error("inspect"); + }, + ownKeys: () => { + throw new Error("enumerate"); + }, + }); +} + +describe("telemetry schemas", () => { + it("preserves serializable definitions and infers exact attributes", () => { + const definition = { + version: 1, + spans: { + operation: { + description: "Test operation", + parents: { kind: "any" }, + startAttributes: { + kind: { type: "string", required: true, values: ["read", "write"], description: "Kind" }, + }, + endAttributes: {}, + events: { + result: { + description: "Result", + attributes: { + outcome: { type: "string", required: true, values: ["ok", "error"], description: "Outcome" }, + }, + }, + }, + status: { default: "ok", errorWhen: "The operation fails" }, + }, + }, + } as const; + const schema = defineTelemetrySchema(definition); + expect(schema).toBe(definition); + expect(() => JSON.stringify(schema)).not.toThrow(); + expectTypeOf>().toMatchTypeOf<{ + kind: "read" | "write"; + }>(); + + const compileTimeFailures = (span: SchemaTelemetrySpan) => { + span.addEvent("result", { outcome: "ok" }); + // @ts-expect-error required event attributes cannot be omitted + span.addEvent("result"); + // @ts-expect-error closed-set event values are exact + span.addEvent("result", { outcome: "other" }); + // @ts-expect-error undeclared events are rejected + span.addEvent("unknown", {}); + // @ts-expect-error empty end schemas reject every attribute + span.setAttributes({ unknown: true }); + }; + expectTypeOf(compileTimeFailures).toBeFunction(); + }); +}); + +describe("NOOP_TELEMETRY_CONTEXT", () => { + it("admits callbacks synchronously and reuses one inert span", async () => { + let admitted = false; + let firstSpan: TelemetrySpan | undefined; + const result = NOOP_TELEMETRY_CONTEXT.startSpan({ name: "first" }, async (span) => { + admitted = true; + firstSpan = span; + const child = await span.startSpan({ name: "child" }, (childSpan) => childSpan); + expect(child).toBe(span); + return 42; + }); + + expect(admitted).toBe(true); + expect(await result).toBe(42); + expect(Object.isFrozen(firstSpan)).toBe(true); + }); + + it("preserves synchronous and asynchronous rejection values", async () => { + const syncError = { kind: "sync" }; + const sync = NOOP_TELEMETRY_CONTEXT.startSpan({ name: "sync" }, () => { + throw syncError; + }); + await expect(sync).rejects.toBe(syncError); + + const asyncError = { kind: "async" }; + const asyncResult = NOOP_TELEMETRY_CONTEXT.startSpan({ name: "async" }, async () => { + throw asyncError; + }); + await expect(asyncResult).rejects.toBe(asyncError); + }); + + it("does not inspect or retain telemetry payloads", async () => { + const options = unreadable({ name: "operation", attributes: { secret: "prompt content" } }); + await NOOP_TELEMETRY_CONTEXT.startSpan(options, (span) => { + const attributes = unreadable({ secret: "content" }); + const status = unreadable({ status: "ok" }); + expect(() => span.addEvent("event", attributes)).not.toThrow(); + expect(() => span.setAttributes(attributes)).not.toThrow(); + expect(() => span.setStatus(status)).not.toThrow(); + }); + }); +}); diff --git a/packages/telemetry/tsconfig.build.json b/packages/telemetry/tsconfig.build.json new file mode 100644 index 000000000..695dd9ad2 --- /dev/null +++ b/packages/telemetry/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "**/*.d.ts", "src/**/*.d.ts"] +} diff --git a/scripts/local-release.mjs b/scripts/local-release.mjs index bbfecb103..7f0c0052d 100644 --- a/scripts/local-release.mjs +++ b/scripts/local-release.mjs @@ -6,6 +6,7 @@ import { isAbsolute, join, relative, resolve } from "node:path"; import { spawnSync } from "node:child_process"; const packages = [ + { directory: "packages/telemetry", name: "@earendil-works/pi-telemetry" }, { directory: "packages/ai", name: "@earendil-works/pi-ai" }, { directory: "packages/tui", name: "@earendil-works/pi-tui" }, { directory: "packages/agent", name: "@earendil-works/pi-agent-core" }, diff --git a/scripts/profile-coding-agent-node.mjs b/scripts/profile-coding-agent-node.mjs index 44ccf86b6..79d8d6a6b 100644 --- a/scripts/profile-coding-agent-node.mjs +++ b/scripts/profile-coding-agent-node.mjs @@ -279,7 +279,7 @@ async function waitForExit(child, errorPrefix) { } async function runBuild() { - process.stdout.write("Building packages/tui, packages/ai, packages/agent, and packages/coding-agent...\n"); + process.stdout.write("Building packages/tui, packages/telemetry, packages/ai, packages/agent, and packages/coding-agent...\n"); const startedAt = performance.now(); const child = spawn( "npm", @@ -289,6 +289,8 @@ async function runBuild() { "--workspace", "packages/tui", "--workspace", + "packages/telemetry", + "--workspace", "packages/ai", "--workspace", "packages/agent", diff --git a/scripts/publish.mjs b/scripts/publish.mjs index 79c8b0f68..3ad86961b 100644 --- a/scripts/publish.mjs +++ b/scripts/publish.mjs @@ -5,6 +5,7 @@ import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; const packages = [ + { directory: "packages/telemetry", name: "@earendil-works/pi-telemetry" }, { directory: "packages/ai", name: "@earendil-works/pi-ai" }, { directory: "packages/agent", name: "@earendil-works/pi-agent-core" }, { directory: "packages/protocol", name: "@earendil-works/pi-protocol" }, diff --git a/tsconfig.json b/tsconfig.json index 1c985946d..85cd390c3 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,6 +6,7 @@ "moduleResolution": "NodeNext", "paths": { "*": ["./*"], + "@earendil-works/pi-telemetry": ["./packages/telemetry/src/index.ts"], "@earendil-works/pi-ai": ["./packages/ai/src/index.ts"], "@earendil-works/pi-ai/oauth": ["./packages/ai/src/oauth.ts"], "@earendil-works/pi-ai/*": ["./packages/ai/src/*.ts", "./packages/ai/src/providers/*.ts"], diff --git a/vitest.base.ts b/vitest.base.ts index 1ede2a2f3..8796ba7e9 100644 --- a/vitest.base.ts +++ b/vitest.base.ts @@ -2,6 +2,7 @@ import { fileURLToPath } from "node:url"; import { defineConfig } from "vitest/config"; export const workspaceSourcePaths = { + telemetryIndex: fileURLToPath(new URL("./packages/telemetry/src/index.ts", import.meta.url)), aiIndex: fileURLToPath(new URL("./packages/ai/src/index.ts", import.meta.url)), aiCompat: fileURLToPath(new URL("./packages/ai/src/compat.ts", import.meta.url)), aiOAuth: fileURLToPath(new URL("./packages/ai/src/oauth.ts", import.meta.url)), @@ -14,6 +15,7 @@ export const workspaceSourcePaths = { export default defineConfig({ resolve: { alias: [ + { find: /^@earendil-works\/pi-telemetry$/, replacement: workspaceSourcePaths.telemetryIndex }, { find: /^@earendil-works\/pi-ai$/, replacement: workspaceSourcePaths.aiIndex }, { find: /^@earendil-works\/pi-ai\/compat$/, replacement: workspaceSourcePaths.aiCompat }, { find: /^@earendil-works\/pi-ai\/oauth$/, replacement: workspaceSourcePaths.aiOAuth },