diff --git a/package-lock.json b/package-lock.json index 04c889174..44ead3b0b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2128,6 +2128,30 @@ "node": ">= 14" } }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, "node_modules/asn1": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", @@ -2159,6 +2183,32 @@ "js-tokens": "^10.0.0" } }, + "node_modules/autoevals": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/autoevals/-/autoevals-0.3.0.tgz", + "integrity": "sha512-4CEzBVhjVBHvk46s+DBcgmEfOdM+zXEoCkvvDqYO2IWdVpZKUJANfX8BvfE5vfcGy24on9zW21Zf7V9cUJdbfg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.17.1", + "compute-cosine-similarity": "^1.1.0", + "js-levenshtein": "^1.1.6", + "js-yaml": "^4.1.0", + "linear-sum-assignment": "^1.0.7", + "mustache": "^4.2.0", + "openai": "^6.7.0", + "zod-to-json-schema": "3.25.0" + }, + "engines": { + "npm": "please-use-pnpm", + "pnpm": ">=10.27.0", + "yarn": "please-use-pnpm" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + } + }, "node_modules/balanced-match": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", @@ -2206,6 +2256,13 @@ "node": "*" } }, + "node_modules/binary-search": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/binary-search/-/binary-search-1.3.6.tgz", + "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==", + "dev": true, + "license": "CC0-1.0" + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -2338,6 +2395,13 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/cheminfo-types": { + "version": "1.16.0", + "resolved": "https://registry.npmjs.org/cheminfo-types/-/cheminfo-types-1.16.0.tgz", + "integrity": "sha512-j0MiXFnLRVEZg3vUr7SXm6F3HuDNVE5arfQyDMiy1w2fAcuOs3QRgVsw7aOIACvr0aqbvn85cucwt+yOu64IiQ==", + "dev": true, + "license": "MIT" + }, "node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", @@ -2354,6 +2418,38 @@ "node": ">=18" } }, + "node_modules/compute-cosine-similarity": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/compute-cosine-similarity/-/compute-cosine-similarity-1.1.0.tgz", + "integrity": "sha512-FXhNx0ILLjGi9Z9+lglLzM12+0uoTnYkHm7GiadXDAr0HGVLm25OivUS1B/LPkbzzvlcXz/1EvWg9ZYyJSdhTw==", + "dev": true, + "dependencies": { + "compute-dot": "^1.1.0", + "compute-l2norm": "^1.1.0", + "validate.io-array": "^1.0.5", + "validate.io-function": "^1.0.2" + } + }, + "node_modules/compute-dot": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/compute-dot/-/compute-dot-1.1.0.tgz", + "integrity": "sha512-L5Ocet4DdMrXboss13K59OK23GXjiSia7+7Ukc7q4Bl+RVpIXK2W9IHMbWDZkh+JUEvJAwOKRaJDiFUa1LTnJg==", + "dev": true, + "dependencies": { + "validate.io-array": "^1.0.3", + "validate.io-function": "^1.0.2" + } + }, + "node_modules/compute-l2norm": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/compute-l2norm/-/compute-l2norm-1.1.0.tgz", + "integrity": "sha512-6EHh1Elj90eU28SXi+h2PLnTQvZmkkHWySpoFz+WOlVNLz3DQoC4ISUHSV9n5jMxPHtKGJ01F4uu2PsXBB8sSg==", + "dev": true, + "dependencies": { + "validate.io-array": "^1.0.3", + "validate.io-function": "^1.0.2" + } + }, "node_modules/convert-source-map": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", @@ -2665,6 +2761,13 @@ "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", "license": "MIT" }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, "node_modules/fast-glob": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", @@ -2688,6 +2791,23 @@ "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", "license": "Unlicense" }, + "node_modules/fast-uri": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", + "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fastq": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", @@ -2721,6 +2841,13 @@ "node": "^12.20 || >= 14.13" } }, + "node_modules/fft.js": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/fft.js/-/fft.js-4.0.4.tgz", + "integrity": "sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==", + "dev": true, + "license": "MIT" + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -3049,6 +3176,13 @@ "node": ">= 0.10" } }, + "node_modules/is-any-array": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-any-array/-/is-any-array-3.0.0.tgz", + "integrity": "sha512-o4h+tylWykC4BD1vaejp6gDxoM13bwW8FGuNs4yIKpj8xbBJcRxJx8vZpq0dCr7ZDEfeKjmsi/euolKhX6f/ww==", + "dev": true, + "license": "MIT" + }, "node_modules/is-core-module": { "version": "2.16.2", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.2.tgz", @@ -3175,6 +3309,16 @@ "jiti": "lib/jiti-cli.mjs" } }, + "node_modules/js-levenshtein": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/js-levenshtein/-/js-levenshtein-1.1.6.tgz", + "integrity": "sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/js-tokens": { "version": "10.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", @@ -3182,6 +3326,29 @@ "dev": true, "license": "MIT" }, + "node_modules/js-yaml": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -3204,6 +3371,13 @@ "node": ">=16" } }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, "node_modules/jwa": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", @@ -3486,6 +3660,18 @@ "url": "https://opencollective.com/parcel" } }, + "node_modules/linear-sum-assignment": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/linear-sum-assignment/-/linear-sum-assignment-1.0.9.tgz", + "integrity": "sha512-1T2Ek3sxpt2mBHeBFMRJEikiIK/yIOwf+mrxv/DkAU/5ddnCMndZL//hFH7QuHa1tbaQADzsf9t7rkGZKqoFfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "cheminfo-types": "^1.8.1", + "ml-matrix": "^6.12.1", + "ml-spectra-processing": "^14.18.0" + } + }, "node_modules/lodash-es": { "version": "4.18.1", "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", @@ -3626,12 +3812,87 @@ "dev": true, "license": "MIT" }, + "node_modules/ml-array-max": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ml-array-max/-/ml-array-max-2.0.0.tgz", + "integrity": "sha512-QQZ4kENwpWmyNb98UXRDFXrmtIXuXtt1+bSbda/2KA85+F+rrJP8hZk6QOkCQXM2Th9mUDYdq/PNByPdT9ID4A==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-any-array": "^3.0.0" + } + }, + "node_modules/ml-array-min": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ml-array-min/-/ml-array-min-2.0.0.tgz", + "integrity": "sha512-GRj6Ky6sW9vGL6yIjgsHmXZ9YgrdmcQ8nCxPqEGeKc6dkfYg1XDYxGFxADUjNuZyoCd5PUscWAS4N+cFaX6hFg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-any-array": "^3.0.0" + } + }, + "node_modules/ml-array-rescale": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ml-array-rescale/-/ml-array-rescale-2.0.0.tgz", + "integrity": "sha512-2GGtKfSno94/kIloWGvpp/U5Q5vLvLrza+SAaGsLeo6Xj4mEbA6Gqx+oTfZFkxnd1grT2X007HfJNs3T5BsiVg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-any-array": "^3.0.0", + "ml-array-max": "^2.0.0", + "ml-array-min": "^2.0.0" + } + }, + "node_modules/ml-matrix": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ml-matrix/-/ml-matrix-6.15.0.tgz", + "integrity": "sha512-wFa1v6KP8bKp+fj0nYmRs1Pb5K4zRkXGKsOvLinvILENFIADncm4XlOI+S1M7yuACMGfI6cfk0IifDgd4j5xmw==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-any-array": "^3.0.0", + "ml-array-rescale": "^2.0.0" + } + }, + "node_modules/ml-spectra-processing": { + "version": "14.35.1", + "resolved": "https://registry.npmjs.org/ml-spectra-processing/-/ml-spectra-processing-14.35.1.tgz", + "integrity": "sha512-yxTB9YpgNmo53U+c2tG1/uKKUl+CNO3hlp+3utuBlhAqDw3AD9c78gI+1rYz+sykOXV76QtAXuAhFZFZjMvgDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "binary-search": "^1.3.6", + "cheminfo-types": "^1.16.0", + "fft.js": "^4.0.4", + "is-any-array": "^3.0.0", + "ml-matrix": "^6.14.0", + "ml-xsadd": "^3.0.1" + } + }, + "node_modules/ml-xsadd": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ml-xsadd/-/ml-xsadd-3.0.1.tgz", + "integrity": "sha512-Fz2q6dwgzGM8wYKGArTUTZDGa4lQFA2Vi6orjGeTVRy22ZnQFKlJuwS9n8NRviqz1KHAHAzdKJwbnYhdo38uYg==", + "dev": true, + "license": "MIT" + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "dev": true, + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/nan": { "version": "2.27.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.27.0.tgz", @@ -4094,6 +4355,16 @@ "node": ">= 0.10" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/resolve": { "version": "1.22.12", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", @@ -4665,6 +4936,19 @@ "dev": true, "license": "MIT" }, + "node_modules/validate.io-array": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/validate.io-array/-/validate.io-array-1.0.6.tgz", + "integrity": "sha512-DeOy7CnPEziggrOO5CZhVKJw6S3Yi7e9e65R1Nl/RTN1vTQKnzjfvks0/8kQ40FP/dsjRAOd4hxmJ7uLa6vxkg==", + "dev": true, + "license": "MIT" + }, + "node_modules/validate.io-function": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/validate.io-function/-/validate.io-function-1.0.2.tgz", + "integrity": "sha512-LlFybRJEriSuBnUhQyG5bwglhh50EpTL2ul23MPIuR1odjO7XaMLFV8vHGwp7AZciFxtYOeiSCT5st+XSPONiQ==", + "dev": true + }, "node_modules/vite": { "version": "8.0.16", "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.16.tgz", @@ -4980,6 +5264,16 @@ "url": "https://github.com/sponsors/colinhacks" } }, + "node_modules/zod-to-json-schema": { + "version": "3.25.0", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.0.tgz", + "integrity": "sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ==", + "dev": true, + "license": "ISC", + "peerDependencies": { + "zod": "^3.25 || ^4" + } + }, "packages/agent": { "name": "@earendil-works/pi-agent-core", "version": "0.85.1", @@ -5225,6 +5519,7 @@ "@earendil-works/pi-coding-agent": "^0.85.1", "@types/node": "22.19.19", "@vitest-evals/core": "0.15.0", + "autoevals": "0.3.0", "shx": "0.4.0", "typescript": "5.9.3", "vitest": "4.1.9", diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts index 81d6e0dea..3219e0634 100644 --- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts @@ -35,6 +35,7 @@ import { Spacer, setCapabilityOverrides, setKeybindings, + type Terminal, Text, TruncatedText, type TUI, @@ -371,6 +372,8 @@ export interface InteractiveModeOptions { tuiMode?: TuiMode; /** Initial interactive theme setting for this invocation. */ initialThemeSetting?: string; + /** Terminal implementation. Defaults to the current process terminal. */ + terminal?: Terminal; } export class InteractiveMode { @@ -536,6 +539,7 @@ export class InteractiveMode { tuiMode, showHardwareCursor: this.settingsManager.getShowHardwareCursor(), logDirectory: getAgentDir(), + terminal: options.terminal, onRightClickPaste: this.onRightClickPaste, fullscreenCopyOnSelect: this.settingsManager.getFullscreenCopyOnSelect(), }); diff --git a/packages/evals/evals/tui.docs.eval.ts b/packages/evals/evals/tui.docs.eval.ts new file mode 100644 index 000000000..eddd20a97 --- /dev/null +++ b/packages/evals/evals/tui.docs.eval.ts @@ -0,0 +1,264 @@ +import { setTimeout } from "node:timers/promises"; +import { stripVTControlCharacters } from "node:util"; +import type { AssistantMessage } from "@earendil-works/pi-ai"; +import { + type AgentSession, + AgentSessionRuntime, + type AgentSessionServices, + InteractiveMode, + SessionManager, + createAgentSessionFromServices, +} from "@earendil-works/pi-coding-agent"; +import { Levenshtein } from "autoevals"; +import { createJudge, describeEval } from "vitest-evals"; +import { createPiDocumentationEvalHarness, type PiCodingAgentInput } from "../src/harness.ts"; + +const CONTEXT_WINDOW = 272_000; +const BUILT_IN_CONTEXT_PATTERN = /\d+(?:\.\d+)?%\/272k(?: \(auto\))?/; +const CONTEXT_FIXTURES = [ + { percent: 42.2, expectedBar: "████░░░░░░ 42%" }, + { percent: 65, expectedBar: "███████░░░ 65%" }, + { percent: 120, expectedBar: "██████████ 100%" }, +] as const; + +class RecordingTerminal { + readonly columns = 100; + readonly rows = 30; + readonly kittyProtocolActive = true; + private writes: string[] = []; + private resizeHandler?: () => void; + + start(_onInput: (data: string) => void, onResize: () => void): void { + this.resizeHandler = onResize; + } + + stop(): void { + this.resizeHandler = undefined; + } + + async drainInput(_maxMs?: number, _idleMs?: number): Promise {} + + write(data: string): void { + this.writes.push(data); + } + + moveBy(_lines: number): void {} + hideCursor(): void {} + showCursor(): void {} + clearLine(): void {} + clearFromCursor(): void {} + clearScreen(): void {} + setTitle(_title: string): void {} + setProgress(_active: boolean): void {} + + requestResize(): void { + this.resizeHandler?.(); + } + + async takeRenders(): Promise { + await setTimeout(30); + const output = this.writes.join(""); + this.writes = []; + const startMarker = "\x1b[?2026h"; + const endMarker = "\x1b[?2026l"; + const renders: string[] = []; + let offset = 0; + while (true) { + const start = output.indexOf(startMarker, offset); + if (start === -1) return renders; + const contentStart = start + startMarker.length; + const end = output.indexOf(endMarker, contentStart); + if (end === -1) throw new Error("TUI emitted an incomplete synchronized render."); + renders.push(stripVTControlCharacters(output.slice(contentStart, end))); + offset = end + endMarker.length; + } + } +} + +type TuiFooterOutput = { + builtInStatusLine: string | null; + observations: Array<{ + percent: number; + renderedOutput: string; + contextUsage: { tokens: number | null; contextWindow: number; percent: number | null } | null; + }>; + extensionErrors: string[]; +}; + +function lines(output: string): string[] { + return output.split(/[\r\n]+/).filter((line) => line.trim().length > 0); +} + +function createContextMessage(model: NonNullable, percent: number): AssistantMessage { + const tokens = Math.round((model.contextWindow * percent) / 100); + return { + role: "assistant", + content: [{ type: "text", text: "Context usage fixture" }], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: tokens, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: tokens, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + }; +} + +async function inspectContextFooter(session: AgentSession, agentDir: string): Promise { + const activeModel = session.model; + if (!activeModel) throw new Error("TUI footer oracle requires an active model."); + const model = { ...activeModel, contextWindow: CONTEXT_WINDOW }; + const services: AgentSessionServices = { + cwd: session.sessionManager.getCwd(), + agentDir, + modelRuntime: session.modelRuntime, + settingsManager: session.settingsManager, + resourceLoader: session.resourceLoader, + diagnostics: [], + }; + const oracleSession = ( + await createAgentSessionFromServices({ + services, + sessionManager: SessionManager.inMemory(services.cwd), + model, + thinkingLevel: "high", + noTools: "all", + }) + ).session; + const runtime = new AgentSessionRuntime(oracleSession, services, async () => { + throw new Error("The TUI footer oracle does not replace sessions."); + }); + const terminal = new RecordingTerminal(); + const mode = new InteractiveMode(runtime, { tuiMode: "regular", terminal }); + const previousOffline = process.env.PI_OFFLINE; + process.env.PI_OFFLINE = "1"; + try { + oracleSession.sessionManager.appendMessage({ + ...createContextMessage(model, 0), + usage: { + input: 106_000, + output: 5_800, + cacheRead: 606_000, + cacheWrite: 0, + totalTokens: 717_800, + cost: { input: 1.008, output: 0, cacheRead: 0, cacheWrite: 0, total: 1.008 }, + }, + }); + let builtInStatusLine: string | null = null; + const observations: TuiFooterOutput["observations"] = []; + for (const [index, { percent }] of CONTEXT_FIXTURES.entries()) { + oracleSession.agent.state.messages = [createContextMessage(model, percent)]; + if (index === 0) await mode.init(); + else terminal.requestResize(); + const renders = await terminal.takeRenders(); + if (index === 0) { + builtInStatusLine = renders.flatMap(lines).find((line) => BUILT_IN_CONTEXT_PATTERN.test(line)) ?? null; + } + observations.push({ + percent, + renderedOutput: renders.at(-1) ?? "", + contextUsage: oracleSession.getContextUsage() ?? null, + }); + } + return { + builtInStatusLine, + observations, + extensionErrors: session.resourceLoader.getExtensions().errors.map(({ error }) => error), + }; + } finally { + try { + mode.stop("resume-hint"); + } finally { + try { + await runtime.dispose(); + } finally { + if (previousOffline === undefined) delete process.env.PI_OFFLINE; + else process.env.PI_OFFLINE = previousOffline; + } + } + } +} + +const harness = createPiDocumentationEvalHarness({ + output: ({ session, agentDir }) => inspectContextFooter(session, agentDir), +}); + +const contextFooterJudge = createJudge( + "ContextFooterJudge", + async ({ output }) => { + let valid = output.extensionErrors.length === 0; + const similarities: number[] = []; + const contextMarker = "{context}"; + const builtInStatusLine = output.builtInStatusLine?.trim().replace(/\s+/g, " "); + const reference = builtInStatusLine?.replace(BUILT_IN_CONTEXT_PATTERN, contextMarker); + if (!reference?.includes(contextMarker)) valid = false; + for (const [index, { percent, expectedBar }] of CONTEXT_FIXTURES.entries()) { + const observation = output.observations[index]; + if (!observation || observation.percent !== percent) { + valid = false; + continue; + } + if ( + observation.contextUsage?.contextWindow !== CONTEXT_WINDOW || + Math.round((observation.contextUsage?.percent ?? -1) * 10) / 10 !== percent + ) { + valid = false; + } + const actualStatusLine = lines(observation.renderedOutput).find( + (line) => line.includes(expectedBar) || BUILT_IN_CONTEXT_PATTERN.test(line), + ); + if (!actualStatusLine) { + valid = false; + continue; + } + const normalizedStatusLine = actualStatusLine.trim().replace(/\s+/g, " "); + if (!normalizedStatusLine.includes(expectedBar) || BUILT_IN_CONTEXT_PATTERN.test(normalizedStatusLine)) { + valid = false; + continue; + } + for (const { expectedBar: otherBar } of CONTEXT_FIXTURES) { + if (otherBar !== expectedBar && normalizedStatusLine.includes(otherBar)) valid = false; + } + if (reference) { + const result = await Levenshtein({ + expected: reference, + output: normalizedStatusLine.replace(expectedBar, contextMarker), + }); + if (result.score !== null) similarities.push(result.score); + } + } + const retained = similarities.length === CONTEXT_FIXTURES.length ? Math.min(...similarities) : 0; + return { + score: valid ? 1 : 0, + metadata: { + rationale: valid + ? `Footer retained ${Math.round(retained * 100)}% of the built-in status text.` + : "Context progress behavior was incorrect.", + retainedPercent: Math.round(retained * 100), + }, + }; + }, +); + +describeEval( + "Customize the interactive context footer", + { harness, judges: [contextFooterJudge], judgeThreshold: null }, + (it) => { + it("replaces numeric context usage with a progress bar", async ({ run }) => { + await run([ + { + type: "prompt", + content: + "Change the Pi footer replacing the built-in context info, e.g. `42.2%/272k (auto)`, with a ten-cell progress bar to make it show ████░░░░░░ 42% instead. Clamp percentages to the 0-100 range. Just replace the context. Otherwise leave everything else the same.", + }, + { type: "reload" }, + ]); + }); + }, +); diff --git a/packages/evals/package.json b/packages/evals/package.json index 155fa9a05..271e3a16b 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -11,6 +11,7 @@ "test": "vitest run --config vitest.test.config.ts" }, "devDependencies": { + "autoevals": "0.3.0", "@earendil-works/pi-ai": "^0.85.1", "@earendil-works/pi-coding-agent": "^0.85.1", "@types/node": "22.19.19",