From 1247476e6df6e632e553eaa61489a77a12b1e463 Mon Sep 17 00:00:00 2001
From: Armin Ronacher <armin.ronacher@active-4.com>
Date: Wed, 16 Sep 2026 17:47:13 +0200
Subject: [PATCH] fix(coding-agent): update eval prompt section markers

---
 packages/evals/src/harness.ts       | 20 +++++++++++++-------
 packages/evals/test/harness.test.ts | 15 +++++++++------
 2 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/packages/evals/src/harness.ts b/packages/evals/src/harness.ts
index eb193d37e..77ff8055f 100644
--- a/packages/evals/src/harness.ts
+++ b/packages/evals/src/harness.ts
@@ -246,10 +246,10 @@ async function promptAgent(session: AgentSession, input: string, signal: AbortSi
 
 function verifySystemPrompt(systemPrompt: string, options: PiCodingAgentHarnessOptions): void {
 	if (options.expectedPiDocumentation === undefined) return;
-	if (!systemPrompt.includes("\nGuidelines:\n")) {
-		throw new Error(`Pi system prompt lost its guidelines in the ${options.name} eval variant.`);
+	if (!systemPrompt.includes("\n<rules>\n")) {
+		throw new Error(`Pi system prompt lost its rules in the ${options.name} eval variant.`);
 	}
-	const hasDocumentation = systemPrompt.includes("\nPi documentation (read only");
+	const hasDocumentation = systemPrompt.includes("\n<docs>\nPi documentation (read only");
 	if (hasDocumentation !== options.expectedPiDocumentation) {
 		throw new Error(`Pi system prompt does not match the ${options.name} eval variant.`);
 	}
@@ -459,11 +459,17 @@ export function resolveDocumentationVariant(
 }
 
 export function excludePiDocumentation(defaultPrompt: string): string {
-	const documentationStart = defaultPrompt.indexOf("\nPi documentation (read only");
+	const documentationStartMarker = "\n<docs>\n";
+	const documentationEndMarker = "\n</docs>";
+	const documentationStart = defaultPrompt.indexOf(documentationStartMarker);
 	if (documentationStart === -1) throw new Error("Default Pi system prompt has no Pi documentation section.");
-	const cwdStart = defaultPrompt.lastIndexOf("\nCurrent working directory: ");
-	if (cwdStart === -1) throw new Error("Default Pi system prompt has no working-directory section.");
-	return defaultPrompt.slice(0, documentationStart) + defaultPrompt.slice(cwdStart);
+	const documentationEnd = defaultPrompt.indexOf(documentationEndMarker, documentationStart);
+	if (documentationEnd === -1) throw new Error("Default Pi system prompt has no complete Pi documentation section.");
+	const cwdStart = defaultPrompt.lastIndexOf("\n<cwd>\n");
+	if (cwdStart < documentationEnd) throw new Error("Default Pi system prompt has no working-directory section.");
+	return (
+		defaultPrompt.slice(0, documentationStart) + defaultPrompt.slice(documentationEnd + documentationEndMarker.length)
+	);
 }
 
 type DocumentationHarnessOptions = Omit<
diff --git a/packages/evals/test/harness.test.ts b/packages/evals/test/harness.test.ts
index 2ca157d03..b8effa224 100644
--- a/packages/evals/test/harness.test.ts
+++ b/packages/evals/test/harness.test.ts
@@ -73,14 +73,15 @@ describe("documentation variant", () => {
 			cwd: "/workspace",
 			selectedTools: [...DOCUMENTATION_EVAL_TOOLS],
 		});
-		expect(prompt).toContain("\nPi documentation (read only");
-		expect(prompt).toContain("\nGuidelines:\n");
-		expect(prompt).toContain("\nCurrent working directory: /workspace");
+		expect(prompt).toContain("\n<docs>\nPi documentation (read only");
+		expect(prompt).toContain("\n<rules>\n");
+		expect(prompt).toContain("\n<cwd>\n/workspace\n</cwd>");
 		expect(prompt).toContain("docs/models.md");
 
 		const stripped = excludePiDocumentation(prompt);
-		expect(stripped).toContain("\nGuidelines:\n");
-		expect(stripped).toContain("\nCurrent working directory: /workspace");
+		expect(stripped).toContain("\n<rules>\n");
+		expect(stripped).toContain("\n<cwd>\n/workspace\n</cwd>");
+		expect(stripped).not.toContain("<docs>");
 		expect(stripped).not.toContain("Pi documentation");
 		expect(stripped).not.toContain("docs/models.md");
 		expect(stripped).not.toContain(getReadmePath());
@@ -90,7 +91,9 @@ describe("documentation variant", () => {
 
 	it("fails closed when prompt markers are missing", () => {
 		expect(() => excludePiDocumentation("Instructions")).toThrow("no Pi documentation section");
-		expect(() => excludePiDocumentation("\nPi documentation (read only\n")).toThrow("no working-directory section");
+		expect(() => excludePiDocumentation("\n<docs>\nPi documentation\n</docs>")).toThrow(
+			"no working-directory section",
+		);
 	});
 
 	it("rejects documentation harnesses outside the container sandbox", () => {