mirror of
https://github.com/badlogic/pi-mono.git
synced 2026-11-05 23:37:25 +00:00
9628c84809
## Summary - add live documentation comparisons for project-scoped and user-scoped `/current-time` prompt templates - require the generated command to expand to the exact requested prompt text - select the requested case from multi-case Vitest reports so skipped siblings do not invalidate observations - strengthen documentation-audit evidence and move the expensive audit behind an explicit `eval:audit` command - remove redundant eval oracles and test-only exports found during the eval-suite audit ## Testing - `npm run check` - `./test.sh` - `npm --prefix packages/evals test` (54 passed, also covered by `./test.sh`) - failed-status mutation: focused report test passed before the fixture repair, failed after it, then passed with production restored - prompt-template documentation eval, three repetitions per variant (6/6 pairs eligible) - project scope: with docs 66.7%, without docs 100%; negative delta and treatment flakiness - user scope: with docs 100%, without docs 100%; no lift - model documentation eval (1/1 pair eligible; treatment 100%, control 0%) - representative live documentation audit (`cli.md` passed)
23 lines
709 B
JSON
23 lines
709 B
JSON
{
|
|
"name": "@earendil-works/pi-evals",
|
|
"version": "1.4.0",
|
|
"private": true,
|
|
"type": "module",
|
|
"scripts": {
|
|
"clean": "shx rm -rf .eval",
|
|
"eval": "npm run eval:host && npm run eval:docs --",
|
|
"eval:host": "vitest run --config vitest.evals.config.ts --project host",
|
|
"eval:audit": "vitest run --config vitest.evals.config.ts --project audit",
|
|
"eval:docs": "node --experimental-strip-types src/cli.ts",
|
|
"test": "vitest run --config vitest.test.config.ts"
|
|
},
|
|
"devDependencies": {
|
|
"@earendil-works/pi-ai": "1.4.0",
|
|
"@earendil-works/pi-coding-agent": "1.4.0",
|
|
"@types/node": "22.19.19",
|
|
"@vitest-evals/core": "0.15.0",
|
|
"shx": "0.3.4",
|
|
"vitest": "4.1.12",
|
|
"vitest-evals": "0.15.0"
|
|
}
|
|
}
|