mirror of
https://github.com/badlogic/pi-mono.git
synced 2026-11-05 23:37:25 +00:00
9628c84809
## Summary - add live documentation comparisons for project-scoped and user-scoped `/current-time` prompt templates - require the generated command to expand to the exact requested prompt text - select the requested case from multi-case Vitest reports so skipped siblings do not invalidate observations - strengthen documentation-audit evidence and move the expensive audit behind an explicit `eval:audit` command - remove redundant eval oracles and test-only exports found during the eval-suite audit ## Testing - `npm run check` - `./test.sh` - `npm --prefix packages/evals test` (54 passed, also covered by `./test.sh`) - failed-status mutation: focused report test passed before the fixture repair, failed after it, then passed with production restored - prompt-template documentation eval, three repetitions per variant (6/6 pairs eligible) - project scope: with docs 66.7%, without docs 100%; negative delta and treatment flakiness - user scope: with docs 100%, without docs 100%; no lift - model documentation eval (1/1 pair eligible; treatment 100%, control 0%) - representative live documentation audit (`cli.md` passed)
56 lines
1.3 KiB
TypeScript
56 lines
1.3 KiB
TypeScript
import { defineConfig, mergeConfig } from "vitest/config";
|
|
import baseConfig, { workspaceSourcePaths } from "../../vitest.base.ts";
|
|
|
|
const evalConfig = defineConfig({
|
|
test: {
|
|
watch: false,
|
|
passWithNoTests: false,
|
|
pool: "forks",
|
|
maxWorkers: 1,
|
|
fileParallelism: false,
|
|
projects: [
|
|
{
|
|
extends: true,
|
|
test: {
|
|
name: "docs",
|
|
include: ["evals/**/*.docs.eval.ts"],
|
|
sequence: { concurrent: false },
|
|
testTimeout: 300_000,
|
|
hookTimeout: 300_000,
|
|
},
|
|
},
|
|
{
|
|
extends: true,
|
|
test: {
|
|
name: "host",
|
|
include: ["evals/**/*.eval.ts"],
|
|
exclude: ["evals/**/*.docs.eval.ts", "evals/documentation-audit.eval.ts"],
|
|
sequence: { concurrent: false },
|
|
testTimeout: 300_000,
|
|
hookTimeout: 300_000,
|
|
},
|
|
},
|
|
{
|
|
extends: true,
|
|
test: {
|
|
name: "audit",
|
|
include: ["evals/documentation-audit.eval.ts"],
|
|
sequence: { concurrent: false },
|
|
testTimeout: 300_000,
|
|
hookTimeout: 300_000,
|
|
},
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
const localConfig = mergeConfig(
|
|
baseConfig,
|
|
defineConfig({
|
|
resolve: {
|
|
alias: [{ find: /^@earendil-works\/pi-coding-agent$/, replacement: workspaceSourcePaths.codingAgentIndex }],
|
|
},
|
|
}),
|
|
);
|
|
|
|
export default process.env.PI_EVAL_CONTAINER === "1" ? evalConfig : mergeConfig(localConfig, evalConfig);
|