From 936e5f40aede9327b9142b63506ed2b68bc6b4ac Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Thu, 22 Oct 2026 23:32:38 +0200 Subject: [PATCH] fix(evals): drop autoevals dependency autoevals@0.3.0 declares engines npm: please-use-pnpm, so every npm install printed an EBADENGINE warning. 0.4.1 removes that but adds a pnpm-enforcing preinstall script. The eval only used the Levenshtein scorer, so inline an equivalent normalized similarity helper instead. --- package-lock.json | 314 -------------------------- packages/evals/evals/tui.docs.eval.ts | 25 +- packages/evals/package.json | 1 - 3 files changed, 19 insertions(+), 321 deletions(-) diff --git a/package-lock.json b/package-lock.json index f90a85222..62665c62f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2995,30 +2995,6 @@ "node": ">= 14" } }, - "node_modules/ajv": { - "version": "8.20.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", - "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, - "license": "Python-2.0" - }, "node_modules/asn1": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", @@ -3038,32 +3014,6 @@ "node": ">=12" } }, - "node_modules/autoevals": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/autoevals/-/autoevals-0.3.0.tgz", - "integrity": "sha512-4CEzBVhjVBHvk46s+DBcgmEfOdM+zXEoCkvvDqYO2IWdVpZKUJANfX8BvfE5vfcGy24on9zW21Zf7V9cUJdbfg==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "dependencies": { - "ajv": "^8.17.1", - "compute-cosine-similarity": "^1.1.0", - "js-levenshtein": "^1.1.6", - "js-yaml": "^4.1.0", - "linear-sum-assignment": "^1.0.7", - "mustache": "^4.2.0", - "openai": "^6.7.0", - "zod-to-json-schema": "3.25.0" - }, - "engines": { - "npm": "please-use-pnpm", - "pnpm": ">=10.27.0", - "yarn": "please-use-pnpm" - }, - "peerDependencies": { - "zod": "^3.25.0 || ^4.0.0" - } - }, "node_modules/balanced-match": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", @@ -3111,13 +3061,6 @@ "node": "*" } }, - "node_modules/binary-search": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/binary-search/-/binary-search-1.3.6.tgz", - "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==", - "dev": true, - "license": "CC0-1.0" - }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -3244,13 +3187,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/cheminfo-types": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/cheminfo-types/-/cheminfo-types-1.16.0.tgz", - "integrity": "sha512-j0MiXFnLRVEZg3vUr7SXm6F3HuDNVE5arfQyDMiy1w2fAcuOs3QRgVsw7aOIACvr0aqbvn85cucwt+yOu64IiQ==", - "dev": true, - "license": "MIT" - }, "node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", @@ -3274,38 +3210,6 @@ "node": ">=18" } }, - "node_modules/compute-cosine-similarity": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/compute-cosine-similarity/-/compute-cosine-similarity-1.1.0.tgz", - "integrity": "sha512-FXhNx0ILLjGi9Z9+lglLzM12+0uoTnYkHm7GiadXDAr0HGVLm25OivUS1B/LPkbzzvlcXz/1EvWg9ZYyJSdhTw==", - "dev": true, - "dependencies": { - "compute-dot": "^1.1.0", - "compute-l2norm": "^1.1.0", - "validate.io-array": "^1.0.5", - "validate.io-function": "^1.0.2" - } - }, - "node_modules/compute-dot": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/compute-dot/-/compute-dot-1.1.0.tgz", - "integrity": "sha512-L5Ocet4DdMrXboss13K59OK23GXjiSia7+7Ukc7q4Bl+RVpIXK2W9IHMbWDZkh+JUEvJAwOKRaJDiFUa1LTnJg==", - "dev": true, - "dependencies": { - "validate.io-array": "^1.0.3", - "validate.io-function": "^1.0.2" - } - }, - "node_modules/compute-l2norm": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/compute-l2norm/-/compute-l2norm-1.1.0.tgz", - "integrity": "sha512-6EHh1Elj90eU28SXi+h2PLnTQvZmkkHWySpoFz+WOlVNLz3DQoC4ISUHSV9n5jMxPHtKGJ01F4uu2PsXBB8sSg==", - "dev": true, - "dependencies": { - "validate.io-array": "^1.0.3", - "validate.io-function": "^1.0.2" - } - }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -3567,36 +3471,12 @@ "node": ">=18" } }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, - "license": "MIT" - }, "node_modules/fast-sha256": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", "license": "Unlicense" }, - "node_modules/fast-uri": { - "version": "3.1.8", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", - "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "BSD-3-Clause" - }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -3638,13 +3518,6 @@ "node": "^12.20 || >= 14.13" } }, - "node_modules/fft.js": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/fft.js/-/fft.js-4.0.4.tgz", - "integrity": "sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==", - "dev": true, - "license": "MIT" - }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -3989,13 +3862,6 @@ "node": ">= 0.10" } }, - "node_modules/is-any-array": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-any-array/-/is-any-array-3.0.0.tgz", - "integrity": "sha512-o4h+tylWykC4BD1vaejp6gDxoM13bwW8FGuNs4yIKpj8xbBJcRxJx8vZpq0dCr7ZDEfeKjmsi/euolKhX6f/ww==", - "dev": true, - "license": "MIT" - }, "node_modules/is-core-module": { "version": "2.17.0", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.17.0.tgz", @@ -4027,39 +3893,6 @@ "jiti": "lib/jiti-cli.mjs" } }, - "node_modules/js-levenshtein": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/js-levenshtein/-/js-levenshtein-1.1.6.tgz", - "integrity": "sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/js-yaml": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", - "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/puzrin" - }, - { - "type": "github", - "url": "https://github.com/sponsors/nodeca" - } - ], - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -4082,13 +3915,6 @@ "node": ">=16" } }, - "node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true, - "license": "MIT" - }, "node_modules/jwa": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", @@ -4393,18 +4219,6 @@ "url": "https://opencollective.com/parcel" } }, - "node_modules/linear-sum-assignment": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/linear-sum-assignment/-/linear-sum-assignment-1.0.9.tgz", - "integrity": "sha512-1T2Ek3sxpt2mBHeBFMRJEikiIK/yIOwf+mrxv/DkAU/5ddnCMndZL//hFH7QuHa1tbaQADzsf9t7rkGZKqoFfQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "cheminfo-types": "^1.8.1", - "ml-matrix": "^6.12.1", - "ml-spectra-processing": "^14.18.0" - } - }, "node_modules/lodash-es": { "version": "4.18.1", "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", @@ -4533,87 +4347,12 @@ "dev": true, "license": "MIT" }, - "node_modules/ml-array-max": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ml-array-max/-/ml-array-max-2.0.0.tgz", - "integrity": "sha512-QQZ4kENwpWmyNb98UXRDFXrmtIXuXtt1+bSbda/2KA85+F+rrJP8hZk6QOkCQXM2Th9mUDYdq/PNByPdT9ID4A==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-any-array": "^3.0.0" - } - }, - "node_modules/ml-array-min": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ml-array-min/-/ml-array-min-2.0.0.tgz", - "integrity": "sha512-GRj6Ky6sW9vGL6yIjgsHmXZ9YgrdmcQ8nCxPqEGeKc6dkfYg1XDYxGFxADUjNuZyoCd5PUscWAS4N+cFaX6hFg==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-any-array": "^3.0.0" - } - }, - "node_modules/ml-array-rescale": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ml-array-rescale/-/ml-array-rescale-2.0.0.tgz", - "integrity": "sha512-2GGtKfSno94/kIloWGvpp/U5Q5vLvLrza+SAaGsLeo6Xj4mEbA6Gqx+oTfZFkxnd1grT2X007HfJNs3T5BsiVg==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-any-array": "^3.0.0", - "ml-array-max": "^2.0.0", - "ml-array-min": "^2.0.0" - } - }, - "node_modules/ml-matrix": { - "version": "6.15.0", - "resolved": "https://registry.npmjs.org/ml-matrix/-/ml-matrix-6.15.0.tgz", - "integrity": "sha512-wFa1v6KP8bKp+fj0nYmRs1Pb5K4zRkXGKsOvLinvILENFIADncm4XlOI+S1M7yuACMGfI6cfk0IifDgd4j5xmw==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-any-array": "^3.0.0", - "ml-array-rescale": "^2.0.0" - } - }, - "node_modules/ml-spectra-processing": { - "version": "14.35.1", - "resolved": "https://registry.npmjs.org/ml-spectra-processing/-/ml-spectra-processing-14.35.1.tgz", - "integrity": "sha512-yxTB9YpgNmo53U+c2tG1/uKKUl+CNO3hlp+3utuBlhAqDw3AD9c78gI+1rYz+sykOXV76QtAXuAhFZFZjMvgDw==", - "dev": true, - "license": "MIT", - "dependencies": { - "binary-search": "^1.3.6", - "cheminfo-types": "^1.16.0", - "fft.js": "^4.0.4", - "is-any-array": "^3.0.0", - "ml-matrix": "^6.14.0", - "ml-xsadd": "^3.0.1" - } - }, - "node_modules/ml-xsadd": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ml-xsadd/-/ml-xsadd-3.0.1.tgz", - "integrity": "sha512-Fz2q6dwgzGM8wYKGArTUTZDGa4lQFA2Vi6orjGeTVRy22ZnQFKlJuwS9n8NRviqz1KHAHAzdKJwbnYhdo38uYg==", - "dev": true, - "license": "MIT" - }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, - "node_modules/mustache": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", - "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", - "dev": true, - "license": "MIT", - "bin": { - "mustache": "bin/mustache" - } - }, "node_modules/nan": { "version": "2.27.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.27.0.tgz", @@ -4729,25 +4468,6 @@ "wrappy": "1" } }, - "node_modules/openai": { - "version": "6.40.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-6.40.0.tgz", - "integrity": "sha512-MWtTjd/gQt4jpbji61NTgFWJLoY/PdRJ6wG9/ZDRMYNMlBKrCrSlkLI+KgHP1vR1qT6LKSAyAqIxno6lcK9JiA==", - "dev": true, - "license": "Apache-2.0", - "peerDependencies": { - "ws": "^8.18.0", - "zod": "^3.25 || ^4.0" - }, - "peerDependenciesMeta": { - "ws": { - "optional": true - }, - "zod": { - "optional": true - } - } - }, "node_modules/p-retry": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", @@ -5049,16 +4769,6 @@ "node": ">= 0.10" } }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/resolve": { "version": "1.22.12", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", @@ -5615,19 +5325,6 @@ "dev": true, "license": "MIT" }, - "node_modules/validate.io-array": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/validate.io-array/-/validate.io-array-1.0.6.tgz", - "integrity": "sha512-DeOy7CnPEziggrOO5CZhVKJw6S3Yi7e9e65R1Nl/RTN1vTQKnzjfvks0/8kQ40FP/dsjRAOd4hxmJ7uLa6vxkg==", - "dev": true, - "license": "MIT" - }, - "node_modules/validate.io-function": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/validate.io-function/-/validate.io-function-1.0.2.tgz", - "integrity": "sha512-LlFybRJEriSuBnUhQyG5bwglhh50EpTL2ul23MPIuR1odjO7XaMLFV8vHGwp7AZciFxtYOeiSCT5st+XSPONiQ==", - "dev": true - }, "node_modules/vite": { "version": "8.3.4", "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.4.tgz", @@ -5999,16 +5696,6 @@ "url": "https://github.com/sponsors/colinhacks" } }, - "node_modules/zod-to-json-schema": { - "version": "3.25.0", - "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.0.tgz", - "integrity": "sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ==", - "dev": true, - "license": "ISC", - "peerDependencies": { - "zod": "^3.25 || ^4" - } - }, "packages/agent": { "name": "@earendil-works/pi-agent-core", "version": "1.4.0", @@ -6324,7 +6011,6 @@ "@earendil-works/pi-coding-agent": "1.4.0", "@types/node": "22.19.19", "@vitest-evals/core": "0.15.0", - "autoevals": "0.3.0", "shx": "0.3.4", "vitest": "4.1.12", "vitest-evals": "0.15.0" diff --git a/packages/evals/evals/tui.docs.eval.ts b/packages/evals/evals/tui.docs.eval.ts index eddd20a97..71988d3a4 100644 --- a/packages/evals/evals/tui.docs.eval.ts +++ b/packages/evals/evals/tui.docs.eval.ts @@ -9,7 +9,6 @@ import { SessionManager, createAgentSessionFromServices, } from "@earendil-works/pi-coding-agent"; -import { Levenshtein } from "autoevals"; import { createJudge, describeEval } from "vitest-evals"; import { createPiDocumentationEvalHarness, type PiCodingAgentInput } from "../src/harness.ts"; @@ -21,6 +20,22 @@ const CONTEXT_FIXTURES = [ { percent: 120, expectedBar: "██████████ 100%" }, ] as const; +/** Normalized Levenshtein similarity: 1 for identical strings, 0 for completely different ones. */ +function levenshteinSimilarity(a: string, b: string): number { + const maxLength = Math.max(a.length, b.length); + if (maxLength === 0) return 1; + let previous = Array.from({ length: b.length + 1 }, (_, index) => index); + for (let i = 1; i <= a.length; i++) { + const current = [i]; + for (let j = 1; j <= b.length; j++) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1; + current[j] = Math.min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + cost); + } + previous = current; + } + return 1 - previous[b.length] / maxLength; +} + class RecordingTerminal { readonly columns = 100; readonly rows = 30; @@ -226,11 +241,9 @@ const contextFooterJudge = createJudge( if (otherBar !== expectedBar && normalizedStatusLine.includes(otherBar)) valid = false; } if (reference) { - const result = await Levenshtein({ - expected: reference, - output: normalizedStatusLine.replace(expectedBar, contextMarker), - }); - if (result.score !== null) similarities.push(result.score); + similarities.push( + levenshteinSimilarity(normalizedStatusLine.replace(expectedBar, contextMarker), reference), + ); } } const retained = similarities.length === CONTEXT_FIXTURES.length ? Math.min(...similarities) : 0; diff --git a/packages/evals/package.json b/packages/evals/package.json index c33b9f58a..b36546214 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -15,7 +15,6 @@ "@earendil-works/pi-coding-agent": "1.4.0", "@types/node": "22.19.19", "@vitest-evals/core": "0.15.0", - "autoevals": "0.3.0", "shx": "0.3.4", "vitest": "4.1.12", "vitest-evals": "0.15.0"