diff --git a/packages/evals/src/vitest-evals/harness-table.ts b/packages/evals/src/vitest-evals/harness-table.ts index 1195e799e..500a39a54 100644 --- a/packages/evals/src/vitest-evals/harness-table.ts +++ b/packages/evals/src/vitest-evals/harness-table.ts @@ -61,34 +61,35 @@ function createSeededRandom(seed: number): () => number { }; } -function serializeString(value: string): string { - const serialized = JSON.stringify(value); - if (serialized === undefined) throw new TypeError("Eval input must be JSON-serializable."); - return serialized; -} - -function canonicalJson(value: unknown, ancestors: WeakSet): string { - if (value === null) return "null"; - if (typeof value === "string") return serializeString(value); - if (typeof value === "boolean") return String(value); +function canonicalizeJson(value: unknown, ancestors: WeakSet): JsonValue { + if (value === null || typeof value === "string" || typeof value === "boolean") return value; if (typeof value === "number") { if (!Number.isFinite(value)) throw new TypeError("Eval input must contain only finite numbers."); - return JSON.stringify(value); + return value; } if (typeof value !== "object") throw new TypeError("Eval input must be JSON-serializable."); if (ancestors.has(value)) throw new TypeError("Eval input must not contain circular references."); ancestors.add(value); try { - if (Array.isArray(value)) return `[${Array.from(value, (item) => canonicalJson(item, ancestors)).join(",")}]`; + if (Array.isArray(value)) { + const result: JsonValue[] = []; + for (let index = 0; index < value.length; index += 1) { + if (!Object.hasOwn(value, index)) throw new TypeError("Eval input arrays must not be sparse."); + result.push(canonicalizeJson(value[index], ancestors)); + } + return result; + } const prototype = Object.getPrototypeOf(value); if (prototype !== Object.prototype && prototype !== null) { throw new TypeError("Eval input must contain only plain objects and arrays."); } - return `{${Object.entries(value) - .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) - .map(([key, item]) => `${serializeString(key)}:${canonicalJson(item, ancestors)}`) - .join(",")}}`; + const entries: Array<[string, unknown]> = Object.entries(value); + return Object.fromEntries( + entries + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) + .map(([key, item]): [string, JsonValue] => [key, canonicalizeJson(item, ancestors)]), + ); } finally { ancestors.delete(value); } @@ -99,7 +100,9 @@ function deriveInputKey(input: unknown): string { const id = input.id; if (typeof id === "string" && id.trim()) return id.trim(); } - return createHash("sha256").update(canonicalJson(input, new WeakSet())).digest("hex"); + const canonicalInput = JSON.stringify(canonicalizeJson(input, new WeakSet())); + if (canonicalInput === undefined) throw new TypeError("Eval input must be JSON-serializable."); + return createHash("sha256").update(canonicalInput).digest("hex"); } export function deriveEvalGroupKey(input: unknown, repetition: number, seed: number): string { diff --git a/packages/evals/test/vitest-evals/harness-table.test.ts b/packages/evals/test/vitest-evals/harness-table.test.ts index ca951727d..6e3c30e4b 100644 --- a/packages/evals/test/vitest-evals/harness-table.test.ts +++ b/packages/evals/test/vitest-evals/harness-table.test.ts @@ -18,13 +18,14 @@ describe("deriveEvalGroupKey", () => { ); expect(deriveEvalGroupKey({ first: 1 }, 1, 42)).not.toBe(deriveEvalGroupKey({ first: 2 }, 1, 42)); expect(deriveEvalGroupKey({ first: 1 }, 1, 42)).not.toBe(deriveEvalGroupKey({ first: 1 }, 2, 42)); + expect(deriveEvalGroupKey(["first", "second"], 1, 42)).not.toBe(deriveEvalGroupKey(["second", "first"], 1, 42)); }); it("rejects non-JSON and circular input", () => { const circular: { self?: unknown } = {}; circular.self = circular; expect(() => deriveEvalGroupKey(new Date(0), 1, 42)).toThrow("only plain objects and arrays"); - expect(() => deriveEvalGroupKey(Array(1), 1, 42)).toThrow("must be JSON-serializable"); + expect(() => deriveEvalGroupKey(Array(1), 1, 42)).toThrow("must not be sparse"); expect(() => deriveEvalGroupKey(circular, 1, 42)).toThrow("must not contain circular references"); }); });