sscoble/pi-mono
1
0
Fork 0
mirror of https://github.com/badlogic/pi-mono.git synced 2026-11-05 23:37:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

fix(coding-agent): update eval prompt section markers

Browse source
This commit is contained in:
Armin Ronacher 2026-09-16 17:47:13 +02:00
parent 9e05370b29
commit 1247476e6d
2 changed files with 22 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

20
packages/evals/src/harness.ts
View file

@ -246,10 +246,10 @@ async function promptAgent(session: AgentSession, input: string, signal: AbortSi
function verifySystemPrompt(systemPrompt: string, options: PiCodingAgentHarnessOptions): void {
if (options.expectedPiDocumentation === undefined) return;
if (!systemPrompt.includes("\nGuidelines:\n")) {
throw new Error(`Pi system prompt lost its guidelines in the ${options.name} eval variant.`);
if (!systemPrompt.includes("\n<rules>\n")) {
throw new Error(`Pi system prompt lost its rules in the ${options.name} eval variant.`);
}
const hasDocumentation = systemPrompt.includes("\nPi documentation (read only");
const hasDocumentation = systemPrompt.includes("\n<docs>\nPi documentation (read only");
if (hasDocumentation !== options.expectedPiDocumentation) {
throw new Error(`Pi system prompt does not match the ${options.name} eval variant.`);
}
@ -459,11 +459,17 @@ export function resolveDocumentationVariant(
}
export function excludePiDocumentation(defaultPrompt: string): string {
const documentationStart = defaultPrompt.indexOf("\nPi documentation (read only");
const documentationStartMarker = "\n<docs>\n";
const documentationEndMarker = "\n</docs>";
const documentationStart = defaultPrompt.indexOf(documentationStartMarker);
if (documentationStart === -1) throw new Error("Default Pi system prompt has no Pi documentation section.");
const cwdStart = defaultPrompt.lastIndexOf("\nCurrent working directory: ");
if (cwdStart === -1) throw new Error("Default Pi system prompt has no working-directory section.");
return defaultPrompt.slice(0, documentationStart) + defaultPrompt.slice(cwdStart);
const documentationEnd = defaultPrompt.indexOf(documentationEndMarker, documentationStart);
if (documentationEnd === -1) throw new Error("Default Pi system prompt has no complete Pi documentation section.");
const cwdStart = defaultPrompt.lastIndexOf("\n<cwd>\n");
if (cwdStart < documentationEnd) throw new Error("Default Pi system prompt has no working-directory section.");
return (
defaultPrompt.slice(0, documentationStart) + defaultPrompt.slice(documentationEnd + documentationEndMarker.length)
);
}
type DocumentationHarnessOptions = Omit<

15
packages/evals/test/harness.test.ts
View file

@ -73,14 +73,15 @@ describe("documentation variant", () => {
cwd: "/workspace",
selectedTools: [...DOCUMENTATION_EVAL_TOOLS],
});
expect(prompt).toContain("\nPi documentation (read only");
expect(prompt).toContain("\nGuidelines:\n");
expect(prompt).toContain("\nCurrent working directory: /workspace");
expect(prompt).toContain("\n<docs>\nPi documentation (read only");
expect(prompt).toContain("\n<rules>\n");
expect(prompt).toContain("\n<cwd>\n/workspace\n</cwd>");
expect(prompt).toContain("docs/models.md");
const stripped = excludePiDocumentation(prompt);
expect(stripped).toContain("\nGuidelines:\n");
expect(stripped).toContain("\nCurrent working directory: /workspace");
expect(stripped).toContain("\n<rules>\n");
expect(stripped).toContain("\n<cwd>\n/workspace\n</cwd>");
expect(stripped).not.toContain("<docs>");
expect(stripped).not.toContain("Pi documentation");
expect(stripped).not.toContain("docs/models.md");
expect(stripped).not.toContain(getReadmePath());
@ -90,7 +91,9 @@ describe("documentation variant", () => {
it("fails closed when prompt markers are missing", () => {
expect(() => excludePiDocumentation("Instructions")).toThrow("no Pi documentation section");
expect(() => excludePiDocumentation("\nPi documentation (read only\n")).toThrow("no working-directory section");
expect(() => excludePiDocumentation("\n<docs>\nPi documentation\n</docs>")).toThrow(
"no working-directory section",
);
});
it("rejects documentation harnesses outside the container sandbox", () => {