feat(schema): benchmark against ata-validator, Ajv and Zod

packages/schema/test/bench/all.ts

@@ -1,5 +1,6 @@
 // Runs every benchmark, each in its own process, and writes the results as JSON next to the printed tables.
 // Speed runs twice: normally and with code generation from strings disabled, as under a strict CSP.
+// The children run with ATA_NO_NATIVE=1 (see candidates.ts).
 // Run: npm run bench [-- --out <file>]   (default: pi-schema-bench-results.json in the temp directory)
 
 import { execFileSync } from "node:child_process";
@@ -28,7 +29,10 @@ const results: Record<string, unknown> = {
 for (const [key, file, flags] of runs) {
 	const json = join(tmpdir(), `pi-schema-bench-${key}.json`);
 	console.log(`\n## ${key}${flags.length > 0 ? ` (${flags.join(" ")})` : ""}\n`);
-	execFileSync(process.execPath, [...flags, join(dir, file), "--json", json], { stdio: "inherit" });
+	execFileSync(process.execPath, [...flags, join(dir, file), "--json", json], {
+		stdio: "inherit",
+		env: { ...process.env, ATA_NO_NATIVE: "1" },
+	});
 	results[key] = JSON.parse(readFileSync(json, "utf8"));
 	rmSync(json);
 }

packages/schema/test/bench/candidates.ts

@@ -1,18 +1,28 @@
 // The validators the benchmarks compare, each behind the same interface: compile a schema, then check values.
 // Every library is used through its fastest boolean path and keeps the object it returns, as an application would.
+// Run the benchmarks through `npm run bench`, which sets ATA_NO_NATIVE=1: ata-validator's optional native addon only
+// serves its buffer APIs, and the verdicts measured here are JavaScript either way.
 
 import { Validator as CfValidator } from "@cfworker/json-schema";
 import { fromJsonSchema } from "@oh-my-pi/omptype";
+import Ajv7Module from "ajv";
+import Ajv2020Module from "ajv/dist/2020.js";
+import draft07MetaSchema from "ajv/dist/refs/json-schema-draft-07.json" with { type: "json" };
+import addFormatsModule from "ajv-formats";
+import { Validator as AtaValidator } from "ata-validator";
 import { Compile } from "typebox/compile";
 import { ResetExternal, ResetFunctions } from "typebox/schema";
 import { Settings } from "typebox/system";
 import { compile, type FormatCheck, type JsonSchema } from "../../src/index.ts";
+import { nativeCases, nativeZod, zodFromJsonSchema } from "./zod.ts";
 
 /** How a benchmark compiles a schema. */
 export interface PrepareOptions {
 	readonly draft: "2020-12" | "7";
 	readonly schemas?: Readonly<Record<string, JsonSchema>>;
 	readonly formats?: Readonly<Record<string, FormatCheck>>;
+	/** The schema is a JSON-Schema-Test-Suite schema (for libraries that take native schemas otherwise). */
+	readonly suite?: boolean;
 }
 
 /** A compiled schema: its boolean check and the object the library returned (kept alive by the caller). */
@@ -92,8 +102,102 @@ const omptype: Candidate = {
 	},
 };
 
+// Ajv's CommonJS modules export the class as module.exports and as `default`; the latter is what its types describe.
+const Ajv2020 = Ajv2020Module.default;
+const Ajv7 = Ajv7Module.default;
+const addFormats = addFormatsModule.default;
+type AjvInstance = InstanceType<typeof Ajv2020>;
+
+// One Ajv per draft, as an application keeps it. Ajv2020 also gets the draft-07 meta-schema, so tool schemas that
+// declare draft-07 compile. strict: false so schemas written for other tools compile; logger: false keeps Ajv from
+// printing generated code when compiling fails without eval.
+function newAjv(draft: "2020-12" | "7", formats: boolean): AjvInstance {
+	const options = { strict: false, validateFormats: formats, logger: false as const };
+	const ajv = draft === "7" ? new Ajv7(options) : new Ajv2020(options);
+	if (draft !== "7") ajv.addMetaSchema(draft07MetaSchema);
+	if (formats) addFormats(ajv);
+	return ajv;
+}
+
+const sharedAjv = new Map<string, AjvInstance>();
+
+// Ajv compiles with new Function and has no interpreter: without eval, compile() throws EvalError.
+const ajv: Candidate = {
+	name: noEval ? "ajv (no eval: fails)" : "ajv",
+	prepare(schema, options) {
+		let instance: AjvInstance;
+		if (options.schemas === undefined) {
+			const key = `${options.draft} ${options.formats !== undefined}`;
+			instance = sharedAjv.get(key) ?? newAjv(options.draft, options.formats !== undefined);
+			sharedAjv.set(key, instance);
+		} else {
+			// The suite reuses $ids across groups, which one shared instance would reject: a fresh one per schema.
+			instance = newAjv(options.draft, options.formats !== undefined);
+			for (const [uri, remote] of Object.entries(options.schemas)) {
+				// Ajv ships its own meta-schemas; adding them again under the same id throws.
+				if (instance.getSchema(uri) === undefined) {
+					try {
+						instance.addSchema(remote as object, uri);
+					} catch {
+						// A remote whose $id Ajv already registered under another key.
+					}
+				}
+			}
+		}
+		const validator = instance.compile(schema as object);
+		return { check: (v) => validator(v) as boolean, validator };
+	},
+};
+
+// ata-validator: its defaults fill `default` values into the input and assert `format`; both are turned off to match
+// the other candidates (format checks only in the format run, with its own checks). It reads the draft only from
+// $schema, so a draft 7 schema without one gets it declared.
+function ataValidator(schema: JsonSchema, options: PrepareOptions, engine: "interpreter" | undefined) {
+	const declared =
+		options.draft === "7" && typeof schema === "object" && !("$schema" in schema)
+			? { $schema: "http://json-schema.org/draft-07/schema#", ...schema }
+			: schema;
+	return new AtaValidator(declared, {
+		useDefaults: false,
+		assertFormat: options.formats !== undefined,
+		schemas: (options.schemas ?? {}) as Record<string, object>,
+		logger: false,
+		...(engine === undefined ? {} : { engine }),
+	});
+}
+
+// Generated code, or its interpreted engine where code generation is blocked.
+const ata: Candidate = {
+	name: noEval ? "ata (no eval: interpreter)" : "ata",
+	prepare(schema, options) {
+		const validator = ataValidator(schema, options, undefined);
+		return { check: (v) => validator.isValidObject(v), validator };
+	},
+};
+
+// Its eval-free engine, a tree of closures, chosen explicitly: what it runs on for schemas from outside.
+const ataInterpreter: Candidate = {
+	name: "ata interpreter",
+	prepare(schema, options) {
+		const validator = ataValidator(schema, options, "interpreter");
+		return { check: (v) => validator.isValidObject(v), validator };
+	},
+};
+
+// Zod: native Zod schemas for the speed cases and the corpus, z.fromJSONSchema only for the suite. Its object parser
+// generates code with new Function when it can and parses without it otherwise; safeParse is its boolean path.
+const zod: Candidate = {
+	name: noEval ? "zod (no eval: no JIT)" : "zod",
+	prepare(schema, options) {
+		const validator = options.suite
+			? zodFromJsonSchema(schema, options.draft)
+			: (nativeCases.get(schema) ?? nativeZod(schema as Record<string, unknown>));
+		return { check: (v) => validator.safeParse(v).success, validator };
+	},
+};
+
 /** Every candidate, ours first. */
-export const candidates: readonly Candidate[] = [piSchema, typebox, cfworker, omptype];
+export const candidates: readonly Candidate[] = [piSchema, typebox, ajv, ata, ataInterpreter, zod, cfworker, omptype];
 
 /** Candidates by name, including TypeBox's interpreter (what TypeBox runs when eval is unavailable). */
 export const byName: ReadonlyMap<string, Candidate> = new Map(

packages/schema/test/bench/conformance.ts

@@ -1,6 +1,7 @@
 // Conformance of every candidate on the vendored JSON-Schema-Test-Suite (drafts 2020-12 and 7), by section:
 // core, optional and optional/format. Every candidate gets the remote documents and meta-schemas; format tests run
-// with pi-schema's format checks (the other libraries check formats on their own or not at all).
+// with pi-schema's format checks (the other libraries check formats on their own or not at all). Zod goes through
+// z.fromJSONSchema, which refuses some keywords and every remote $ref: a schema it refuses fails all its tests.
 // Run: node test/bench/conformance.ts [--json <file>]
 
 import { writeFileSync } from "node:fs";
@@ -29,6 +30,7 @@ for (const file of suiteFiles()) {
 					draft: file.draft,
 					schemas: suiteSchemas(),
 					formats: kind === "format" ? formats : undefined,
+					suite: true,
 				}).check;
 			} catch {
 				continue; // A schema the library cannot compile fails all its tests.

packages/schema/test/bench/size.ts

@@ -1,4 +1,5 @@
-// Bundle size of what a validator needs from each library: esbuild bundles and minifies the import, then gzip -9.
+// Bundle size of what a validator needs from each library, as a browser app ships it: esbuild bundles and minifies the
+// import for the browser platform (one file, no code splitting), then gzip -9.
 // Run: node test/bench/size.ts [--json <file>]
 
 import { writeFileSync } from "node:fs";
@@ -12,6 +13,13 @@ const entries: Record<string, string> = {
 	"pi-schema + formats": `export { compile } from ${JSON.stringify(`${src}index.ts`)}; export { formats } from ${JSON.stringify(`${src}formats/index.ts`)};`,
 	typebox: `export { Compile } from "typebox/compile";`,
 	cfworker: `export { Validator } from "@cfworker/json-schema";`,
+	ajv: `export { default } from "ajv/dist/2020.js";`,
+	"ajv + ajv-formats": `export { default } from "ajv/dist/2020.js"; export { default as addFormats } from "ajv-formats";`,
+	"ata-validator": `export { Validator } from "ata-validator";`,
+	// Zod's documented `import * as z` (a named `{ z }` import keeps every locale, about 90 KB gzipped). Its core with
+	// native schemas, and with z.fromJSONSchema for JSON Schema.
+	zod: `import * as z from "zod"; export const check = (v: unknown) => z.object({ a: z.string() }).safeParse(v).success;`,
+	"zod + fromJSONSchema": `import * as z from "zod"; export const fromJSONSchema = z.fromJSONSchema;`,
 	omptype: `export { fromJsonSchema } from "@oh-my-pi/omptype";`,
 };
 
@@ -25,8 +33,7 @@ for (const [name, contents] of Object.entries(entries)) {
 		bundle: true,
 		minify: true,
 		format: "esm",
-		platform: "neutral",
-		mainFields: ["module", "main"],
+		platform: "browser",
 		write: false,
 		logLevel: "silent",
 	});

packages/schema/test/bench/speed.ts

@@ -20,21 +20,37 @@ function median(values: number[]): number {
 	return [...values].sort((a, b) => a - b)[values.length >> 1];
 }
 
-/** Median ns per check, by case name, input ("valid"/"invalid") and candidate name. */
-export type SpeedResults = Record<string, Record<"valid" | "invalid", Record<string, number>>>;
+/**
+ * Median ns per check, by case name, input ("valid"/"invalid") and candidate name; null where the candidate cannot
+ * compile the schema (Ajv without eval).
+ */
+export type SpeedResults = Record<string, Record<"valid" | "invalid", Record<string, number | null>>>;
 
 const results: SpeedResults = {};
 for (const c of cases) {
 	results[c.name] = { valid: {}, invalid: {} };
 	for (const input of ["valid", "invalid"] as const) {
 		const value = c[input];
-		const runs = candidates.map((candidate) => {
-			const { check } = candidate.prepare(c.schema, { draft: "2020-12" });
+		const runs: {
+			candidate: (typeof candidates)[number];
+			check: (v: unknown) => boolean;
+			n: number;
+			samples: number[];
+		}[] = [];
+		for (const candidate of candidates) {
+			let check: (v: unknown) => boolean;
+			try {
+				check = candidate.prepare(c.schema, { draft: "2020-12" }).check;
+			} catch (error) {
+				if (!(error instanceof EvalError)) throw error;
+				results[c.name][input][candidate.name] = null;
+				continue;
+			}
 			if (check(value) !== (input === "valid")) throw new Error(`${candidate.name} gets ${c.name} (${input}) wrong`);
 			let n = 1000;
 			while (time(check, value, n) * n < ROUND_MS * 1e6 && n < 1e8) n *= 2;
-			return { candidate, check, n, samples: [] as number[] };
-		});
+			runs.push({ candidate, check, n, samples: [] });
+		}
 		for (let round = 0; round < ROUNDS; round++) {
 			for (let i = 0; i < runs.length; i++) {
 				const run = runs[(i + round) % runs.length];
@@ -57,6 +73,7 @@ const out = process.argv.indexOf("--json");
 if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t"));
 if (sink < 0) console.log(sink);
 
-function format(ns: number): string {
+function format(ns: number | null): string {
+	if (ns === null) return "fails";
 	return ns >= 1000 ? `${(ns / 1000).toFixed(ns >= 10000 ? 0 : 1)} µs` : `${ns.toFixed(0)} ns`;
 }

packages/schema/test/bench/zod.ts

@@ -0,0 +1,187 @@
+// Zod in the comparison. Zod is not a JSON Schema validator: the speed cases are written here as the native Zod
+// schemas a developer would write (z.strictObject for closed objects, z.discriminatedUnion for tagged unions, z.lazy
+// for recursion), and the corpus schemas are built into the same native calls by `nativeZod`. Only the conformance run
+// goes through z.fromJSONSchema, Zod's (semi-experimental) JSON Schema import.
+
+import * as z from "zod";
+import type { JsonSchema } from "../../src/index.ts";
+import { cases } from "./cases.ts";
+
+const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+const block = z.discriminatedUnion("kind", [
+	z.strictObject({ kind: z.literal("text"), text: z.string() }),
+	z.strictObject({
+		kind: z.literal("image"),
+		src: z.string(),
+		width: z.number().int().min(0),
+		height: z.number().int().min(0),
+		alt: z.string().optional(),
+	}),
+	z.strictObject({ kind: z.literal("code"), lang: z.enum(["ts", "js", "py"]), source: z.string() }),
+]);
+
+const id = z.number().int().min(0);
+
+const range = z.strictObject({ start: z.number().int().min(1), end: z.number().int().min(1) });
+const replace = z.strictObject({ oldText: z.string(), newText: z.string() });
+const lineEdit = z.strictObject({ range, text: z.string() });
+
+const richText = z.strictObject({
+	type: z.enum(["text", "mention"]),
+	content: z.string().max(2000),
+	annotations: z.strictObject({ bold: z.boolean(), italic: z.boolean(), color: z.string() }).optional(),
+});
+
+type PageBlock = {
+	type: "paragraph" | "heading_1" | "heading_2" | "bulleted_list_item" | "to_do";
+	rich_text: z.infer<typeof richText>[];
+	checked?: boolean;
+	children?: PageBlock[];
+};
+
+const pageBlock: z.ZodType<PageBlock> = z.lazy(() =>
+	z.strictObject({
+		type: z.enum(["paragraph", "heading_1", "heading_2", "bulleted_list_item", "to_do"]),
+		rich_text: z.array(richText),
+		checked: z.boolean().optional(),
+		children: z.array(pageBlock).optional(),
+	}),
+);
+
+/** The speed cases as native Zod schemas, by case name (the JSON Schema cases in cases.ts, one for one). */
+const native: Record<string, z.ZodType> = {
+	"small closed object": z.strictObject({ text: z.string() }),
+	"nested object with tagged unions": z.strictObject({
+		id: z.string().regex(uuid),
+		title: z.string().min(1).max(200),
+		tags: z.array(z.string()).max(16),
+		author: z.strictObject({ name: z.string(), email: z.string().optional() }),
+		blocks: z.array(block),
+		score: z.number().min(0).max(1).optional(),
+		draft: z.boolean().optional(),
+	}),
+	"message union of 5": z.discriminatedUnion("kind", [
+		z.strictObject({
+			kind: z.literal("hello"),
+			protocol: z.number().int().min(0),
+			incarnation: z.string().regex(uuid),
+			calls: z.number().int().min(1),
+		}),
+		z.strictObject({
+			kind: z.literal("hold"),
+			hold: id,
+			type: z.string(),
+			version: z.number().int().min(1),
+			fingerprint: z.string().regex(/^[0-9a-f]{16}$/),
+			params: z.unknown(),
+			address: z.strictObject({ kind: z.string(), params: z.unknown() }),
+			chain: z.array(z.string().regex(uuid)).max(64).optional(),
+		}),
+		z.strictObject({ kind: z.literal("release"), hold: id }),
+		z.strictObject({ kind: z.literal("call"), call: id, hold: id, method: id, args: z.unknown().optional() }),
+		z.strictObject({ kind: z.literal("abort"), call: id }),
+	]),
+	"tool: read": z.object({ path: z.string(), offset: z.number().optional(), limit: z.number().optional() }),
+	"tool: edit": z.object({ path: z.string(), edits: z.array(z.object({ oldText: z.string(), newText: z.string() })) }),
+	"tool: bash": z.object({ command: z.string(), timeout: z.number().optional() }),
+	"MCP create_issue (pattern, anyOf, enum)": z.strictObject({
+		owner: z
+			.string()
+			.min(1)
+			.regex(/^[A-Za-z0-9-]+$/),
+		repo: z
+			.string()
+			.min(1)
+			.regex(/^[A-Za-z0-9._-]+$/),
+		title: z.string().min(1),
+		body: z.string().optional(),
+		labels: z.array(z.string()).optional(),
+		assignees: z.array(z.string()).optional(),
+		milestone: z.number().nullable().optional(),
+		state: z.enum(["open", "closed"]).optional(),
+	}),
+	"MCP edit_file ($ref, anyOf)": z.strictObject({
+		path: z.string().regex(/^(\/|\.\/|[A-Za-z]:\\)?[^\u0000]+$/),
+		edits: z.array(z.union([replace, lineEdit])).min(1),
+		dryRun: z.boolean().optional(),
+	}),
+	"MCP create_page (recursive $ref, oneOf)": z.strictObject({
+		parent: z.union([
+			z.strictObject({ page_id: z.string().regex(uuid) }),
+			z.strictObject({ database_id: z.string().regex(uuid) }),
+		]),
+		title: z.string().min(1),
+		due: z
+			.string()
+			.regex(/^\d{4}-\d{2}-\d{2}$/)
+			.optional(),
+		children: z.array(pageBlock).max(100).optional(),
+	}),
+};
+
+/** The native Zod schema of each speed case's JSON Schema. */
+export const nativeCases: ReadonlyMap<JsonSchema, z.ZodType> = new Map(
+	cases.map((c) => {
+		const schema = native[c.name];
+		if (schema === undefined) throw new Error(`No native Zod schema for the case ${c.name}`);
+		return [c.schema, schema];
+	}),
+);
+
+type Json = Record<string, unknown>;
+
+/**
+ * The native Zod schema a developer would write for a corpus schema (corpus.ts): objects (closed ones strict),
+ * strings with bounds, patterns and enums, bounded integers, numbers, booleans, string arrays, descriptions, and
+ * $defs reused through their constant. Throws for any other shape.
+ */
+export function nativeZod(schema: Json, root: Json = schema): z.ZodType {
+	if (typeof schema.$ref === "string") {
+		const name = schema.$ref.replace("#/$defs/", "");
+		return nativeZod((root.$defs as Record<string, Json>)[name], root);
+	}
+	const built = nativeType(schema, root);
+	return typeof schema.description === "string" ? built.describe(schema.description) : built;
+}
+
+function nativeType(schema: Json, root: Json): z.ZodType {
+	switch (schema.type) {
+		case "object": {
+			const required = new Set(schema.required as string[]);
+			const shape: Record<string, z.ZodType> = {};
+			for (const [key, value] of Object.entries(schema.properties as Record<string, Json>)) {
+				const property = nativeZod(value, root);
+				shape[key] = required.has(key) ? property : property.optional();
+			}
+			return schema.additionalProperties === false ? z.strictObject(shape) : z.object(shape);
+		}
+		case "string": {
+			if (Array.isArray(schema.enum)) return z.enum(schema.enum as [string, ...string[]]);
+			let string = z.string();
+			if (typeof schema.minLength === "number") string = string.min(schema.minLength);
+			if (typeof schema.maxLength === "number") string = string.max(schema.maxLength);
+			if (typeof schema.pattern === "string") string = string.regex(new RegExp(schema.pattern));
+			return string;
+		}
+		case "integer":
+			return z
+				.number()
+				.int()
+				.min(schema.minimum as number)
+				.max(schema.maximum as number);
+		case "number":
+			return z.number();
+		case "boolean":
+			return z.boolean();
+		case "array":
+			return z.array(z.string()).max(schema.maxItems as number);
+		default:
+			throw new Error(`No native Zod builder for ${JSON.stringify(schema).slice(0, 80)}`);
+	}
+}
+
+/** z.fromJSONSchema with the draft given; it accepts only local $refs, so remote documents are not passed. */
+export function zodFromJsonSchema(schema: JsonSchema, draft: "2020-12" | "7"): z.ZodType {
+	return z.fromJSONSchema(schema as never, { defaultTarget: draft === "7" ? "draft-7" : "draft-2020-12" });
+}