feat(coding-agent): add vitest eval harness (#7085)

scripts/sync-versions.js

@@ -1,29 +1,47 @@
 #!/usr/bin/env node
 
 /**
- * Syncs all non-private workspace package dependency versions to match their current versions.
- * This ensures release packages, including unpublished packages, use lockstep versioning.
+ * Validates lockstep versions for published packages, then synchronizes
+ * internal dependency versions in all workspace packages, including private ones.
  */
 
 import { readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { findPackageDirectories } from "./package-workspaces.mjs";
 
-const packages = findPackageDirectories()
+const GENERATED_PACKAGE_SUFFIXES = [join("coding-agent", "install-lock")];
+
+function synchronizedDependencyVersion(dependencyName, currentSpecifier, versionMap) {
+	const directVersion = versionMap.get(dependencyName);
+	if (directVersion) {
+		return `^${directVersion}`;
+	}
+	if (typeof currentSpecifier !== "string" || !currentSpecifier.startsWith("npm:")) {
+		return null;
+	}
+
+	const separator = currentSpecifier.lastIndexOf("@");
+	const aliasPackage = currentSpecifier.slice("npm:".length, separator);
+	const aliasVersion = versionMap.get(aliasPackage);
+	return aliasVersion ? `npm:${aliasPackage}@${aliasVersion}` : null;
+}
+
+const packageRoot = process.argv[2] ?? "packages";
+const workspacePackages = findPackageDirectories(packageRoot)
+	.filter((directory) => !GENERATED_PACKAGE_SUFFIXES.some((suffix) => directory.endsWith(suffix)))
 	.map((directory) => {
 		const path = join(directory, "package.json");
 		return { data: JSON.parse(readFileSync(path, "utf8")), path };
-	})
-	.filter((pkg) => pkg.data.private !== true);
-
-const versionMap = new Map(packages.map((pkg) => [pkg.data.name, pkg.data.version]));
+	});
+const publishedPackages = workspacePackages.filter((pkg) => pkg.data.private !== true);
+const versionMap = new Map(workspacePackages.map((pkg) => [pkg.data.name, pkg.data.version]));
 
 console.log("Current versions:");
-for (const [name, version] of [...versionMap].sort(([a], [b]) => a.localeCompare(b))) {
-	console.log(`  ${name}: ${version}`);
+for (const pkg of [...publishedPackages].sort((a, b) => a.data.name.localeCompare(b.data.name))) {
+	console.log(`  ${pkg.data.name}: ${pkg.data.version}`);
 }
 
-const versions = new Set(versionMap.values());
+const versions = new Set(publishedPackages.map((pkg) => pkg.data.version));
 if (versions.size > 1) {
 	console.error("\nERROR: Not all non-private packages have the same version.");
 	console.error("Expected lockstep versioning. Run one of:");
@@ -36,37 +54,33 @@ if (versions.size > 1) {
 console.log("\nAll non-private packages are at the same version (lockstep).");
 
 let totalUpdates = 0;
-for (const pkg of packages) {
-	let updated = false;
-
+const updatedPackages = new Set();
+for (const pkg of workspacePackages) {
 	for (const dependencyType of ["dependencies", "devDependencies"]) {
 		const dependencies = pkg.data[dependencyType];
 		if (!dependencies) {
 			continue;
 		}
 
-		for (const [dependencyName, currentVersion] of Object.entries(dependencies)) {
-			const dependencyVersion = versionMap.get(dependencyName);
-			if (!dependencyVersion) {
-				continue;
-			}
-
-			const newVersion = `^${dependencyVersion}`;
-			if (currentVersion === newVersion) {
+		for (const [dependencyName, currentSpecifier] of Object.entries(dependencies)) {
+			const newSpecifier = synchronizedDependencyVersion(dependencyName, currentSpecifier, versionMap);
+			if (!newSpecifier || currentSpecifier === newSpecifier) {
 				continue;
 			}
 
 			console.log(`\n${pkg.data.name}:`);
-			console.log(`  ${dependencyName}: ${currentVersion} → ${newVersion}${dependencyType === "devDependencies" ? " (devDependencies)" : ""}`);
-			dependencies[dependencyName] = newVersion;
-			updated = true;
+			console.log(
+				`  ${dependencyName}: ${currentSpecifier} → ${newSpecifier}${dependencyType === "devDependencies" ? " (devDependencies)" : ""}`,
+			);
+			dependencies[dependencyName] = newSpecifier;
+			updatedPackages.add(pkg);
 			totalUpdates++;
 		}
 	}
+}
 
-	if (updated) {
-		writeFileSync(pkg.path, `${JSON.stringify(pkg.data, null, "\t")}\n`);
-	}
+for (const pkg of updatedPackages) {
+	writeFileSync(pkg.path, `${JSON.stringify(pkg.data, null, "\t")}\n`);
 }
 
 if (totalUpdates === 0) {

scripts/sync-versions.test.mjs

@@ -0,0 +1,75 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import test from "node:test";
+
+const syncVersionsScript = fileURLToPath(new URL("./sync-versions.js", import.meta.url));
+
+async function writeManifest(root, relativeDirectory, manifest) {
+	const directory = join(root, relativeDirectory);
+	await mkdir(directory, { recursive: true });
+	await writeFile(join(directory, "package.json"), `${JSON.stringify(manifest, null, "\t")}\n`);
+}
+
+async function readManifest(root, relativeDirectory) {
+	return JSON.parse(await readFile(join(root, relativeDirectory, "package.json"), "utf8"));
+}
+
+function runSyncVersions(root) {
+	return spawnSync(process.execPath, [syncVersionsScript, join(root, "packages")], {
+		cwd: root,
+		encoding: "utf8",
+	});
+}
+
+test("synchronizes private dependencies without touching generated manifests or relaxing published lockstep", async () => {
+	const root = await mkdtemp(join(tmpdir(), "pi-sync-versions-"));
+	try {
+		await writeManifest(root, "packages/ai", {
+			name: "@earendil-works/pi-ai",
+			version: "2.0.0",
+		});
+		await writeManifest(root, "packages/coding-agent", {
+			name: "@earendil-works/pi-coding-agent",
+			version: "2.0.0",
+		});
+		await writeManifest(root, "packages/evals", {
+			name: "@earendil-works/pi-evals",
+			version: "9.9.9",
+			private: true,
+			dependencies: {
+				"@earendil-works/pi-coding-agent": "^1.0.0",
+				"@mariozechner/pi-ai": "npm:@earendil-works/pi-ai@1.0.0",
+			},
+		});
+		await writeManifest(root, "packages/coding-agent/install-lock", {
+			name: "generated-install-lock",
+			version: "0.0.0",
+			private: true,
+			dependencies: {
+				"@earendil-works/pi-coding-agent": "^1.0.0",
+			},
+		});
+
+		const result = runSyncVersions(root);
+		assert.equal(result.status, 0, result.stderr);
+
+		const evalsManifest = await readManifest(root, "packages/evals");
+		assert.equal(evalsManifest.dependencies["@earendil-works/pi-coding-agent"], "^2.0.0");
+		assert.equal(evalsManifest.dependencies["@mariozechner/pi-ai"], "npm:@earendil-works/pi-ai@2.0.0");
+		const generatedManifest = await readManifest(root, "packages/coding-agent/install-lock");
+		assert.equal(generatedManifest.dependencies["@earendil-works/pi-coding-agent"], "^1.0.0");
+
+		await writeManifest(root, "packages/ai", {
+			name: "@earendil-works/pi-ai",
+			version: "3.0.0",
+		});
+		const lockstepFailure = runSyncVersions(root);
+		assert.equal(lockstepFailure.status, 1, lockstepFailure.stderr);
+	} finally {
+		await rm(root, { recursive: true, force: true });
+	}
+});