From a6bde34bb61499cfcd9ef70545d5750416a632f7 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sun, 1 Nov 2026 10:22:05 +0000 Subject: [PATCH] feat(schema): benchmark against ata-validator, Ajv and Zod --- package-lock.json | 222 +++++++++++++++++++++- packages/schema/CHANGELOG.md | 2 +- packages/schema/README.md | 128 +++++++++---- packages/schema/package.json | 6 +- packages/schema/test/bench/all.ts | 6 +- packages/schema/test/bench/candidates.ts | 106 ++++++++++- packages/schema/test/bench/conformance.ts | 4 +- packages/schema/test/bench/size.ts | 13 +- packages/schema/test/bench/speed.ts | 31 ++- packages/schema/test/bench/zod.ts | 187 ++++++++++++++++++ 10 files changed, 652 insertions(+), 53 deletions(-) create mode 100644 packages/schema/test/bench/zod.ts diff --git a/package-lock.json b/package-lock.json index 4bc15ba5c..8abc3354d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,6 +50,116 @@ } } }, + "node_modules/@ata-validator/native-darwin-arm64": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-darwin-arm64/-/native-darwin-arm64-1.61.3.tgz", + "integrity": "sha512-DC+iozKe5mNpsmT4MLuMxG2pIeGPVR3YBYuBmops0AF5UG6eSxg/X0484hW/Nmc2dil54+b4N41hIhS5U9rExg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@ata-validator/native-darwin-x64": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-darwin-x64/-/native-darwin-x64-1.61.3.tgz", + "integrity": "sha512-q63+k48mWlvrNaUXmtGVxp2NWTl3wXlkkWkc2q4MerHzjIfeFBca30tfTRcyVXtDXPCj8BRJkrI69aJJXk+pLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@ata-validator/native-linux-arm64-gnu": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-linux-arm64-gnu/-/native-linux-arm64-gnu-1.61.3.tgz", + "integrity": "sha512-D9OG2oKJd/hW9F8IFoqIfK+SDfyryzSGr5YRVJB39LwvvWNNfh6pxsYpjthEjwqGuJdoBtYIqOUFnjgBqDWh9w==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ata-validator/native-linux-arm64-musl": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-linux-arm64-musl/-/native-linux-arm64-musl-1.61.3.tgz", + "integrity": "sha512-L1NhRd5/i6v7VQw28gR5L53+Zlo09CnYfXBapgGUC9I3uNppOw56/dihU5cPrd5T/k9VeljosllOP+dWA9cLpA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ata-validator/native-linux-x64-gnu": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-linux-x64-gnu/-/native-linux-x64-gnu-1.61.3.tgz", + "integrity": "sha512-Dlp+hcvQqKkAQEyrr7RohCbYoLYHdSjeB5puU3Ju92kSeH4MaF451q0VqwOm3StaMQB3LrLA03GDb4/G3b5gsA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ata-validator/native-linux-x64-musl": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-linux-x64-musl/-/native-linux-x64-musl-1.61.3.tgz", + "integrity": "sha512-2TutVxyhzbpUyBFzuGNp4fjyC3Suhi1idFfbk883tPdfcmp2/j0Q5QT5ySk4LPLGrNzV2haqjESZvha/QKctkw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ata-validator/native-win32-x64": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-win32-x64/-/native-win32-x64-1.61.3.tgz", + "integrity": "sha512-Ac2fqD70b/lQSuLDdz03xiGPhn2FK7oKXdQ7JbCq9ZYiJBbymhFazMKkRaI+3Y5XKIV3zalAfNGzUJAzMUsXow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@aws-sdk/client-bedrock-runtime": { "version": "3.1127.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-runtime/-/client-bedrock-runtime-3.1127.0.tgz", @@ -3001,6 +3111,41 @@ "node": ">= 14" } }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/asn1": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", @@ -3020,6 +3165,36 @@ "node": ">=12" } }, + "node_modules/ata-validator": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/ata-validator/-/ata-validator-1.61.3.tgz", + "integrity": "sha512-6o3V8lpiVQzzHe6blbQZj9saD0Qxmhdv2TTyO5BgPcUWeP3EZj572f9w1b93E6c+BmUce0QOf7GvwOffRUf3tw==", + "dev": true, + "license": "MIT", + "bin": { + "ata": "bin/ata.js" + }, + "engines": { + "node": ">=20.0.0" + }, + "optionalDependencies": { + "@ata-validator/native-darwin-arm64": "1.61.3", + "@ata-validator/native-darwin-x64": "1.61.3", + "@ata-validator/native-linux-arm64-gnu": "1.61.3", + "@ata-validator/native-linux-arm64-musl": "1.61.3", + "@ata-validator/native-linux-x64-gnu": "1.61.3", + "@ata-validator/native-linux-x64-musl": "1.61.3", + "@ata-validator/native-win32-x64": "1.61.3" + }, + "peerDependencies": { + "yaml": "^2.0.0" + }, + "peerDependenciesMeta": { + "yaml": { + "optional": true + } + } + }, "node_modules/balanced-match": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", @@ -3451,12 +3626,36 @@ "node": ">=18" } }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, "node_modules/fast-sha256": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", "license": "Unlicense" }, + "node_modules/fast-uri": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.9.tgz", + "integrity": "sha512-hS/0Wt3Ya043oKXs0sIxrMpKQITXhMLIrVAKKXi4gozGC9v7YRWVnXIZzjNPTO02M9BdxLCWqK1yRXgNziVyJQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -3774,6 +3973,13 @@ "node": ">=16" } }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, "node_modules/jwa": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", @@ -4590,6 +4796,16 @@ "node": ">= 6" } }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/retry": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", @@ -5778,8 +5994,12 @@ "devDependencies": { "@cfworker/json-schema": "4.1.1", "@oh-my-pi/omptype": "18.13.3", + "ajv": "8.20.0", + "ajv-formats": "3.0.1", + "ata-validator": "1.61.3", "typebox": "1.3.27", - "vitest": "4.1.12" + "vitest": "4.1.12", + "zod": "4.6.7" }, "engines": { "node": ">=22.19.0" diff --git a/packages/schema/CHANGELOG.md b/packages/schema/CHANGELOG.md index 69650940a..02f86d449 100644 --- a/packages/schema/CHANGELOG.md +++ b/packages/schema/CHANGELOG.md @@ -9,4 +9,4 @@ - `compile(schema, { coerce: true })` adds `coerce(value)` and `parse(value)` for loosely typed input such as a model's tool call arguments: optional `null`s dropped, TypeBox 1.3.27's `Value.Convert` rules keyed on JSON Schema keywords (numeric strings, `"true"`, single values to arrays, `const`/`enum` members, union branches), then Ajv-style primitive coercion by `type`. `coerce` copies only the paths it changes and returns the input itself when nothing changes; `parse` coerces, then checks. Validators compiled without the option have neither; the README lists every rule. - `unresolvedRefs: "reject"` compiles a `$ref` that names nothing as `false` (the error is `unresolved reference `) instead of throwing, for schemas whose references may dangle. - `formats` also checks `url` and `json-pointer-uri-fragment`, as TypeBox and ajv-formats do. -- `npm run bench` compares conformance, speed, setup cost, memory and bundle size with TypeBox, @cfworker/json-schema and omptype. +- `npm run bench` compares conformance, speed (with and without eval), setup cost, memory and bundle size with ata-validator (generated code and its interpreter), Ajv, Zod (native schemas, and `z.fromJSONSchema` for the test suite), TypeBox, @cfworker/json-schema and omptype. diff --git a/packages/schema/README.md b/packages/schema/README.md index 100186dd9..6654fe54d 100644 --- a/packages/schema/README.md +++ b/packages/schema/README.md @@ -1,7 +1,7 @@ # @earendil-works/pi-schema JSON Schema builders and validation without `eval`. A schema compiles once into plain closures, so it checks values -about as fast as validators that generate code, also under a strict Content Security Policy, and needs no +at close to the speed of validators that generate code, also under a strict Content Security Policy, and needs no dependencies. - Drafts 2020-12 and 7. Passes 2,222 of the 2,223 core tests of the official @@ -181,53 +181,107 @@ coerced by any pass. ## Benchmarks -Node 24.4.1 on one core of an i9-13900, against TypeBox 1.3.27, @cfworker/json-schema 4.1.1 and @oh-my-pi/omptype -18.13.3 (`fromJsonSchema`). +Node 24.4.1 on one core of an i9-13900. The other libraries: -| test suite (tests passed) | pi-schema | TypeBox | cfworker | omptype | -|---|---:|---:|---:|---:| -| draft 2020-12 core (1,298) | 1,297 | 1,276 | 1,249 | 863 | -| draft 7 core (925) | 925 | 924 | 918 | 659 | -| draft 2020-12 format (895), with `formats` | 895 | 895 | 740 | 412 | +- ata-validator 1.61.3: its generated code, and its eval-free interpreter chosen with `engine: "interpreter"`; +- Ajv 8.20.0, with ajv-formats 3.0.1 for the format tests; +- Zod 4.6.7: native Zod schemas for speed, setup and memory, and `z.fromJSONSchema` for the test suite; +- TypeBox 1.3.27; +- @cfworker/json-schema 4.1.1; +- @oh-my-pi/omptype 18.13.3, through `fromJsonSchema`. -| check, valid value | pi-schema | TypeBox | cfworker | omptype | -|---|---:|---:|---:|---:| -| nested object with tagged unions | 409 ns | 474 ns | 44 µs | 689 ns | -| union of 5 messages | 89 ns | 95 ns | 10 µs | 93 ns | -| tool schema (read) | 31 ns | 9 ns | 1.9 µs | 9 ns | -| MCP tool schema with `$ref` | 162 ns | 89 ns | 13 µs | 1.8 µs | -| recursive `$ref` schema | 753 ns | 1.2 µs | 41 µs | 7.4 µs | +| test suite (tests passed) | pi-schema | ata | ata interpreter | Ajv | Zod | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| draft 2020-12 core (1,298) | 1,297 | 1,298 | 1,298 | 1,236 | 693 | 1,276 | 1,249 | 863 | +| draft 7 core (925) | 925 | 925 | 925 | 917 | 632 | 924 | 918 | 659 | +| draft 2020-12 format (895), with format checks | 895 | 814 | 814 | 750 | 412 | 895 | 740 | 412 | -| same, without eval (strict CSP) | pi-schema | TypeBox | cfworker | omptype | -|---|---:|---:|---:|---:| -| nested object with tagged unions | 407 ns | 42 µs | 43 µs | 2.6 µs | -| MCP tool schema with `$ref` | 162 ns | 36 µs | 12 µs | 2.0 µs | -| recursive `$ref` schema | 750 ns | 124 µs | 41 µs | 9.0 µs | +| check, valid value | pi-schema | ata | ata interpreter | Ajv | Zod | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| nested object with tagged unions | 410 ns | 180 ns | 836 ns | 627 ns | 1.3 µs | 475 ns | 43 µs | 730 ns | +| union of 5 messages | 86 ns | 37 ns | 212 ns | 70 ns | 297 ns | 95 ns | 11 µs | 96 ns | +| tool schema (read) | 31 ns | 10 ns | 41 ns | 13 ns | 41 ns | 9 ns | 2.0 µs | 9 ns | +| MCP tool schema with `$ref` | 161 ns | 64 ns | 314 ns | 136 ns | 927 ns | 87 ns | 15 µs | 1.9 µs | +| recursive `$ref` schema | 755 ns | 307 ns | 1.9 µs | 378 ns | 2.7 µs | 1.2 µs | 41 µs | 7.5 µs | -| cost | pi-schema | TypeBox | cfworker | omptype | -|---|---:|---:|---:|---:| -| compile a new schema | 21 µs | 203 µs | 20 µs | 26 µs + 106 µs on its first check | -| memory, 3,200 distinct schemas | 7.1 MB | 54.7 MB | 7.6 MB | 20.0 MB | -| bundle, minified + gzip | 8.5 KB | 35.2 KB | 5.8 KB | 38.8 KB | +| same, without eval (strict CSP) | pi-schema | ata | ata interpreter | Ajv | Zod | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| nested object with tagged unions | 411 ns | 1.6 µs | 844 ns | fails | 3.4 µs | 44 µs | 44 µs | 2.7 µs | +| union of 5 messages | 87 ns | 396 ns | 209 ns | fails | 652 ns | 10 µs | 11 µs | 501 ns | +| MCP tool schema with `$ref` | 164 ns | 484 ns | 326 ns | fails | 2.0 µs | 39 µs | 13 µs | 2.2 µs | +| recursive `$ref` schema | 746 ns | 2.8 µs | 1.9 µs | fails | 7.3 µs | 130 µs | 42 µs | 9.0 µs | -Reproduce with `npm run bench` in `packages/schema` (about 5 minutes). It runs every library through its fastest -boolean path and keeps the object each returns. The scripts are in `test/bench/`. +| cost | pi-schema | ata | ata interpreter | Ajv | Zod | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| compile a new schema | 20 µs | 4.9 µs | 1.3 µs | 1.4 ms | 151 µs | 210 µs | 22 µs | 28 µs | +| its first check | 0.8 µs | 335 µs | 87 µs | 195 µs | 466 µs | 41 µs | 7.1 µs | 109 µs | +| memory, 3,200 distinct schemas | 7.1 MB | 36.0 MB | 51.3 MB | 59.7 MB | 170.8 MB | 54.7 MB | 7.6 MB | 19.9 MB | +| bundle, minified + gzip | 8.5 KB | 134.0 KB | 134.0 KB | 39.7 KB | 24.4 KB | 35.2 KB | 5.8 KB | 38.8 KB | -- **conformance:** every library on the vendored test suite, with the remote documents and meta-schemas the tests - reference. -- **speed:** realistic schemas, in ns per check after warm-up; the median of 15 rounds, with the libraries' - order rotated in each round. -- **speed without eval:** the same, run with `--disallow-code-generation-from-strings`. -- **setup:** compiling 1,000 schemas a library has not seen, and their first checks. -- **memory:** 3,200 distinct tool-like schemas, each library in its own process. -- **size:** esbuild bundles of what a validator needs, minified and gzipped. +Where code generation is unavailable: -Where code generation is unavailable, TypeBox falls back to its interpreter, and omptype throws `EvalError` once per -schema, then validates through its interpreter. +- **ata** answers through its interpreted engine without being told. That automatic fallback measured slower than the + engine chosen with `engine: "interpreter"`. +- **Ajv** cannot compile at all: `compile()` throws `EvalError`. +- **Zod** notices and parses objects without its generated code. +- **TypeBox** falls back to its interpreter. +- **omptype** throws `EvalError` once per schema, then validates through its interpreter. + +ata, Zod and omptype compile on a schema's first check, so their compile cost shows there. ata's bundle is for the +browser without code splitting; with splitting, its first load is about 53 KB plus a 76 KB generator chunk loaded +later. + +Zod is not a JSON Schema validator: + +- **Speed, setup and memory:** Zod gets the same schemas written as native Zod schemas (`z.strictObject`, + `z.discriminatedUnion`, `z.lazy`), with `.describe()` for the corpus descriptions, and `safeParse` as the check. + Its memory includes building the schemas, which the other libraries receive as JSON. Without the descriptions it + is 131 MB. +- **Test suite:** through `z.fromJSONSchema`, which is semi-experimental. It refuses `unevaluatedProperties`, + `unevaluatedItems`, `dependentSchemas`, `dependentRequired`, `if`/`then`/`else`, `not`, and every `$ref` that is + not local. A refused schema fails all its tests: 405 of the 1,298 draft 2020-12 core tests and 107 of the 925 draft + 7 core tests. +- **Bundle:** measured with Zod's documented `import * as z from "zod"`. A named `import { z }` keeps every locale + (about 90 KB gzipped), and `z.fromJSONSchema` brings it to 40.0 KB; `zod/mini` is about 6 KB. Flat objects of simple types check in 13–80 ns, against 7–15 ns for TypeBox. Without code generation, reading a property whose name comes from a list costs a few nanoseconds. +### Reproduce + +`npm run bench` in `packages/schema` takes about 12 minutes. It runs every library through its fastest boolean path +(ata's `isValidObject()`) and keeps the object each returns. ata runs with `ATA_NO_NATIVE=1`: its optional native +addon only serves its buffer APIs. The scripts are in `test/bench/`. + +- **Conformance:** every library on the vendored test suite, with the remote documents and meta-schemas the tests + reference (Zod through `z.fromJSONSchema`, which takes no remote documents). ata reads the draft only from `$schema`, so draft 7 schemas without one get it declared. ata's defaults + (`useDefaults`, `assertFormat`) are turned off to match the other libraries; formats are checked only in the + format tests, by each library's own checks. +- **Speed:** realistic schemas, in ns per check after warm-up. Each result is the median of 15 rounds, with the + libraries' order rotated in each round. +- **Speed without eval:** the same, run with `--disallow-code-generation-from-strings`. +- **Setup:** compiling 1,000 schemas a library has not seen, and their first checks. +- **Memory:** 3,200 distinct tool-like schemas, compiled and each checked 3 times, each library in its own process. +- **Size:** esbuild browser bundles of what a validator needs, minified and gzipped. + +### Code generated at build time + +Ajv (`ajv/dist/standalone`) and ata (`ata build`, `ata compile`, `toStandaloneModule()`) can also generate their +validation code when the app is built. The app then imports plain modules and evaluates nothing at run time, so those +checks run at code-generation speed under a strict CSP. This is not in the benchmark, because it is a different +deal: + +- **A build step per schema.** Every schema must be known when the app is built. Schemas that arrive at run time + (MCP servers, extensions, user input) cannot be validated this way. +- **Code per schema.** In a separate measurement on the same 3,200 schemas: + - Ajv: 1.8 ms to build and 8.9 KB of JavaScript per schema; + - ata: 1.1 ms and 13 KB per schema, or 0.3–0.8 KB gzipped when only its `isValid` export is bundled; + - importing all 3,200 generated validators took 40 MB of heap (Ajv, one module) and 68 MB (ata, one module per + schema). + +pi-schema compiles at run time in 25 µs per schema, generates no code, and covers schemas known at build time and +schemas that arrive later. + ## Development `npm test` runs the official test suite (drafts 2020-12 and 7, including the optional and format tests, vendored in diff --git a/packages/schema/package.json b/packages/schema/package.json index f6e94e1d2..3fdb1b14f 100644 --- a/packages/schema/package.json +++ b/packages/schema/package.json @@ -50,7 +50,11 @@ "devDependencies": { "@cfworker/json-schema": "4.1.1", "@oh-my-pi/omptype": "18.13.3", + "ajv": "8.20.0", + "ajv-formats": "3.0.1", + "ata-validator": "1.61.3", "typebox": "1.3.27", - "vitest": "4.1.12" + "vitest": "4.1.12", + "zod": "4.6.7" } } diff --git a/packages/schema/test/bench/all.ts b/packages/schema/test/bench/all.ts index bb8421fc2..96ce485c4 100644 --- a/packages/schema/test/bench/all.ts +++ b/packages/schema/test/bench/all.ts @@ -1,5 +1,6 @@ // Runs every benchmark, each in its own process, and writes the results as JSON next to the printed tables. // Speed runs twice: normally and with code generation from strings disabled, as under a strict CSP. +// The children run with ATA_NO_NATIVE=1 (see candidates.ts). // Run: npm run bench [-- --out ] (default: pi-schema-bench-results.json in the temp directory) import { execFileSync } from "node:child_process"; @@ -28,7 +29,10 @@ const results: Record = { for (const [key, file, flags] of runs) { const json = join(tmpdir(), `pi-schema-bench-${key}.json`); console.log(`\n## ${key}${flags.length > 0 ? ` (${flags.join(" ")})` : ""}\n`); - execFileSync(process.execPath, [...flags, join(dir, file), "--json", json], { stdio: "inherit" }); + execFileSync(process.execPath, [...flags, join(dir, file), "--json", json], { + stdio: "inherit", + env: { ...process.env, ATA_NO_NATIVE: "1" }, + }); results[key] = JSON.parse(readFileSync(json, "utf8")); rmSync(json); } diff --git a/packages/schema/test/bench/candidates.ts b/packages/schema/test/bench/candidates.ts index 8b231c855..ce0c6d4cf 100644 --- a/packages/schema/test/bench/candidates.ts +++ b/packages/schema/test/bench/candidates.ts @@ -1,18 +1,28 @@ // The validators the benchmarks compare, each behind the same interface: compile a schema, then check values. // Every library is used through its fastest boolean path and keeps the object it returns, as an application would. +// Run the benchmarks through `npm run bench`, which sets ATA_NO_NATIVE=1: ata-validator's optional native addon only +// serves its buffer APIs, and the verdicts measured here are JavaScript either way. import { Validator as CfValidator } from "@cfworker/json-schema"; import { fromJsonSchema } from "@oh-my-pi/omptype"; +import Ajv7Module from "ajv"; +import Ajv2020Module from "ajv/dist/2020.js"; +import draft07MetaSchema from "ajv/dist/refs/json-schema-draft-07.json" with { type: "json" }; +import addFormatsModule from "ajv-formats"; +import { Validator as AtaValidator } from "ata-validator"; import { Compile } from "typebox/compile"; import { ResetExternal, ResetFunctions } from "typebox/schema"; import { Settings } from "typebox/system"; import { compile, type FormatCheck, type JsonSchema } from "../../src/index.ts"; +import { nativeCases, nativeZod, zodFromJsonSchema } from "./zod.ts"; /** How a benchmark compiles a schema. */ export interface PrepareOptions { readonly draft: "2020-12" | "7"; readonly schemas?: Readonly>; readonly formats?: Readonly>; + /** The schema is a JSON-Schema-Test-Suite schema (for libraries that take native schemas otherwise). */ + readonly suite?: boolean; } /** A compiled schema: its boolean check and the object the library returned (kept alive by the caller). */ @@ -92,8 +102,102 @@ const omptype: Candidate = { }, }; +// Ajv's CommonJS modules export the class as module.exports and as `default`; the latter is what its types describe. +const Ajv2020 = Ajv2020Module.default; +const Ajv7 = Ajv7Module.default; +const addFormats = addFormatsModule.default; +type AjvInstance = InstanceType; + +// One Ajv per draft, as an application keeps it. Ajv2020 also gets the draft-07 meta-schema, so tool schemas that +// declare draft-07 compile. strict: false so schemas written for other tools compile; logger: false keeps Ajv from +// printing generated code when compiling fails without eval. +function newAjv(draft: "2020-12" | "7", formats: boolean): AjvInstance { + const options = { strict: false, validateFormats: formats, logger: false as const }; + const ajv = draft === "7" ? new Ajv7(options) : new Ajv2020(options); + if (draft !== "7") ajv.addMetaSchema(draft07MetaSchema); + if (formats) addFormats(ajv); + return ajv; +} + +const sharedAjv = new Map(); + +// Ajv compiles with new Function and has no interpreter: without eval, compile() throws EvalError. +const ajv: Candidate = { + name: noEval ? "ajv (no eval: fails)" : "ajv", + prepare(schema, options) { + let instance: AjvInstance; + if (options.schemas === undefined) { + const key = `${options.draft} ${options.formats !== undefined}`; + instance = sharedAjv.get(key) ?? newAjv(options.draft, options.formats !== undefined); + sharedAjv.set(key, instance); + } else { + // The suite reuses $ids across groups, which one shared instance would reject: a fresh one per schema. + instance = newAjv(options.draft, options.formats !== undefined); + for (const [uri, remote] of Object.entries(options.schemas)) { + // Ajv ships its own meta-schemas; adding them again under the same id throws. + if (instance.getSchema(uri) === undefined) { + try { + instance.addSchema(remote as object, uri); + } catch { + // A remote whose $id Ajv already registered under another key. + } + } + } + } + const validator = instance.compile(schema as object); + return { check: (v) => validator(v) as boolean, validator }; + }, +}; + +// ata-validator: its defaults fill `default` values into the input and assert `format`; both are turned off to match +// the other candidates (format checks only in the format run, with its own checks). It reads the draft only from +// $schema, so a draft 7 schema without one gets it declared. +function ataValidator(schema: JsonSchema, options: PrepareOptions, engine: "interpreter" | undefined) { + const declared = + options.draft === "7" && typeof schema === "object" && !("$schema" in schema) + ? { $schema: "http://json-schema.org/draft-07/schema#", ...schema } + : schema; + return new AtaValidator(declared, { + useDefaults: false, + assertFormat: options.formats !== undefined, + schemas: (options.schemas ?? {}) as Record, + logger: false, + ...(engine === undefined ? {} : { engine }), + }); +} + +// Generated code, or its interpreted engine where code generation is blocked. +const ata: Candidate = { + name: noEval ? "ata (no eval: interpreter)" : "ata", + prepare(schema, options) { + const validator = ataValidator(schema, options, undefined); + return { check: (v) => validator.isValidObject(v), validator }; + }, +}; + +// Its eval-free engine, a tree of closures, chosen explicitly: what it runs on for schemas from outside. +const ataInterpreter: Candidate = { + name: "ata interpreter", + prepare(schema, options) { + const validator = ataValidator(schema, options, "interpreter"); + return { check: (v) => validator.isValidObject(v), validator }; + }, +}; + +// Zod: native Zod schemas for the speed cases and the corpus, z.fromJSONSchema only for the suite. Its object parser +// generates code with new Function when it can and parses without it otherwise; safeParse is its boolean path. +const zod: Candidate = { + name: noEval ? "zod (no eval: no JIT)" : "zod", + prepare(schema, options) { + const validator = options.suite + ? zodFromJsonSchema(schema, options.draft) + : (nativeCases.get(schema) ?? nativeZod(schema as Record)); + return { check: (v) => validator.safeParse(v).success, validator }; + }, +}; + /** Every candidate, ours first. */ -export const candidates: readonly Candidate[] = [piSchema, typebox, cfworker, omptype]; +export const candidates: readonly Candidate[] = [piSchema, typebox, ajv, ata, ataInterpreter, zod, cfworker, omptype]; /** Candidates by name, including TypeBox's interpreter (what TypeBox runs when eval is unavailable). */ export const byName: ReadonlyMap = new Map( diff --git a/packages/schema/test/bench/conformance.ts b/packages/schema/test/bench/conformance.ts index d0cc72a7e..9d8977d76 100644 --- a/packages/schema/test/bench/conformance.ts +++ b/packages/schema/test/bench/conformance.ts @@ -1,6 +1,7 @@ // Conformance of every candidate on the vendored JSON-Schema-Test-Suite (drafts 2020-12 and 7), by section: // core, optional and optional/format. Every candidate gets the remote documents and meta-schemas; format tests run -// with pi-schema's format checks (the other libraries check formats on their own or not at all). +// with pi-schema's format checks (the other libraries check formats on their own or not at all). Zod goes through +// z.fromJSONSchema, which refuses some keywords and every remote $ref: a schema it refuses fails all its tests. // Run: node test/bench/conformance.ts [--json ] import { writeFileSync } from "node:fs"; @@ -29,6 +30,7 @@ for (const file of suiteFiles()) { draft: file.draft, schemas: suiteSchemas(), formats: kind === "format" ? formats : undefined, + suite: true, }).check; } catch { continue; // A schema the library cannot compile fails all its tests. diff --git a/packages/schema/test/bench/size.ts b/packages/schema/test/bench/size.ts index b6996af60..46a09e3d3 100644 --- a/packages/schema/test/bench/size.ts +++ b/packages/schema/test/bench/size.ts @@ -1,4 +1,5 @@ -// Bundle size of what a validator needs from each library: esbuild bundles and minifies the import, then gzip -9. +// Bundle size of what a validator needs from each library, as a browser app ships it: esbuild bundles and minifies the +// import for the browser platform (one file, no code splitting), then gzip -9. // Run: node test/bench/size.ts [--json ] import { writeFileSync } from "node:fs"; @@ -12,6 +13,13 @@ const entries: Record = { "pi-schema + formats": `export { compile } from ${JSON.stringify(`${src}index.ts`)}; export { formats } from ${JSON.stringify(`${src}formats/index.ts`)};`, typebox: `export { Compile } from "typebox/compile";`, cfworker: `export { Validator } from "@cfworker/json-schema";`, + ajv: `export { default } from "ajv/dist/2020.js";`, + "ajv + ajv-formats": `export { default } from "ajv/dist/2020.js"; export { default as addFormats } from "ajv-formats";`, + "ata-validator": `export { Validator } from "ata-validator";`, + // Zod's documented `import * as z` (a named `{ z }` import keeps every locale, about 90 KB gzipped). Its core with + // native schemas, and with z.fromJSONSchema for JSON Schema. + zod: `import * as z from "zod"; export const check = (v: unknown) => z.object({ a: z.string() }).safeParse(v).success;`, + "zod + fromJSONSchema": `import * as z from "zod"; export const fromJSONSchema = z.fromJSONSchema;`, omptype: `export { fromJsonSchema } from "@oh-my-pi/omptype";`, }; @@ -25,8 +33,7 @@ for (const [name, contents] of Object.entries(entries)) { bundle: true, minify: true, format: "esm", - platform: "neutral", - mainFields: ["module", "main"], + platform: "browser", write: false, logLevel: "silent", }); diff --git a/packages/schema/test/bench/speed.ts b/packages/schema/test/bench/speed.ts index 6f470951b..ba2f17b23 100644 --- a/packages/schema/test/bench/speed.ts +++ b/packages/schema/test/bench/speed.ts @@ -20,21 +20,37 @@ function median(values: number[]): number { return [...values].sort((a, b) => a - b)[values.length >> 1]; } -/** Median ns per check, by case name, input ("valid"/"invalid") and candidate name. */ -export type SpeedResults = Record>>; +/** + * Median ns per check, by case name, input ("valid"/"invalid") and candidate name; null where the candidate cannot + * compile the schema (Ajv without eval). + */ +export type SpeedResults = Record>>; const results: SpeedResults = {}; for (const c of cases) { results[c.name] = { valid: {}, invalid: {} }; for (const input of ["valid", "invalid"] as const) { const value = c[input]; - const runs = candidates.map((candidate) => { - const { check } = candidate.prepare(c.schema, { draft: "2020-12" }); + const runs: { + candidate: (typeof candidates)[number]; + check: (v: unknown) => boolean; + n: number; + samples: number[]; + }[] = []; + for (const candidate of candidates) { + let check: (v: unknown) => boolean; + try { + check = candidate.prepare(c.schema, { draft: "2020-12" }).check; + } catch (error) { + if (!(error instanceof EvalError)) throw error; + results[c.name][input][candidate.name] = null; + continue; + } if (check(value) !== (input === "valid")) throw new Error(`${candidate.name} gets ${c.name} (${input}) wrong`); let n = 1000; while (time(check, value, n) * n < ROUND_MS * 1e6 && n < 1e8) n *= 2; - return { candidate, check, n, samples: [] as number[] }; - }); + runs.push({ candidate, check, n, samples: [] }); + } for (let round = 0; round < ROUNDS; round++) { for (let i = 0; i < runs.length; i++) { const run = runs[(i + round) % runs.length]; @@ -57,6 +73,7 @@ const out = process.argv.indexOf("--json"); if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); if (sink < 0) console.log(sink); -function format(ns: number): string { +function format(ns: number | null): string { + if (ns === null) return "fails"; return ns >= 1000 ? `${(ns / 1000).toFixed(ns >= 10000 ? 0 : 1)} µs` : `${ns.toFixed(0)} ns`; } diff --git a/packages/schema/test/bench/zod.ts b/packages/schema/test/bench/zod.ts new file mode 100644 index 000000000..c0d4d8371 --- /dev/null +++ b/packages/schema/test/bench/zod.ts @@ -0,0 +1,187 @@ +// Zod in the comparison. Zod is not a JSON Schema validator: the speed cases are written here as the native Zod +// schemas a developer would write (z.strictObject for closed objects, z.discriminatedUnion for tagged unions, z.lazy +// for recursion), and the corpus schemas are built into the same native calls by `nativeZod`. Only the conformance run +// goes through z.fromJSONSchema, Zod's (semi-experimental) JSON Schema import. + +import * as z from "zod"; +import type { JsonSchema } from "../../src/index.ts"; +import { cases } from "./cases.ts"; + +const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +const block = z.discriminatedUnion("kind", [ + z.strictObject({ kind: z.literal("text"), text: z.string() }), + z.strictObject({ + kind: z.literal("image"), + src: z.string(), + width: z.number().int().min(0), + height: z.number().int().min(0), + alt: z.string().optional(), + }), + z.strictObject({ kind: z.literal("code"), lang: z.enum(["ts", "js", "py"]), source: z.string() }), +]); + +const id = z.number().int().min(0); + +const range = z.strictObject({ start: z.number().int().min(1), end: z.number().int().min(1) }); +const replace = z.strictObject({ oldText: z.string(), newText: z.string() }); +const lineEdit = z.strictObject({ range, text: z.string() }); + +const richText = z.strictObject({ + type: z.enum(["text", "mention"]), + content: z.string().max(2000), + annotations: z.strictObject({ bold: z.boolean(), italic: z.boolean(), color: z.string() }).optional(), +}); + +type PageBlock = { + type: "paragraph" | "heading_1" | "heading_2" | "bulleted_list_item" | "to_do"; + rich_text: z.infer[]; + checked?: boolean; + children?: PageBlock[]; +}; + +const pageBlock: z.ZodType = z.lazy(() => + z.strictObject({ + type: z.enum(["paragraph", "heading_1", "heading_2", "bulleted_list_item", "to_do"]), + rich_text: z.array(richText), + checked: z.boolean().optional(), + children: z.array(pageBlock).optional(), + }), +); + +/** The speed cases as native Zod schemas, by case name (the JSON Schema cases in cases.ts, one for one). */ +const native: Record = { + "small closed object": z.strictObject({ text: z.string() }), + "nested object with tagged unions": z.strictObject({ + id: z.string().regex(uuid), + title: z.string().min(1).max(200), + tags: z.array(z.string()).max(16), + author: z.strictObject({ name: z.string(), email: z.string().optional() }), + blocks: z.array(block), + score: z.number().min(0).max(1).optional(), + draft: z.boolean().optional(), + }), + "message union of 5": z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("hello"), + protocol: z.number().int().min(0), + incarnation: z.string().regex(uuid), + calls: z.number().int().min(1), + }), + z.strictObject({ + kind: z.literal("hold"), + hold: id, + type: z.string(), + version: z.number().int().min(1), + fingerprint: z.string().regex(/^[0-9a-f]{16}$/), + params: z.unknown(), + address: z.strictObject({ kind: z.string(), params: z.unknown() }), + chain: z.array(z.string().regex(uuid)).max(64).optional(), + }), + z.strictObject({ kind: z.literal("release"), hold: id }), + z.strictObject({ kind: z.literal("call"), call: id, hold: id, method: id, args: z.unknown().optional() }), + z.strictObject({ kind: z.literal("abort"), call: id }), + ]), + "tool: read": z.object({ path: z.string(), offset: z.number().optional(), limit: z.number().optional() }), + "tool: edit": z.object({ path: z.string(), edits: z.array(z.object({ oldText: z.string(), newText: z.string() })) }), + "tool: bash": z.object({ command: z.string(), timeout: z.number().optional() }), + "MCP create_issue (pattern, anyOf, enum)": z.strictObject({ + owner: z + .string() + .min(1) + .regex(/^[A-Za-z0-9-]+$/), + repo: z + .string() + .min(1) + .regex(/^[A-Za-z0-9._-]+$/), + title: z.string().min(1), + body: z.string().optional(), + labels: z.array(z.string()).optional(), + assignees: z.array(z.string()).optional(), + milestone: z.number().nullable().optional(), + state: z.enum(["open", "closed"]).optional(), + }), + "MCP edit_file ($ref, anyOf)": z.strictObject({ + path: z.string().regex(/^(\/|\.\/|[A-Za-z]:\\)?[^\u0000]+$/), + edits: z.array(z.union([replace, lineEdit])).min(1), + dryRun: z.boolean().optional(), + }), + "MCP create_page (recursive $ref, oneOf)": z.strictObject({ + parent: z.union([ + z.strictObject({ page_id: z.string().regex(uuid) }), + z.strictObject({ database_id: z.string().regex(uuid) }), + ]), + title: z.string().min(1), + due: z + .string() + .regex(/^\d{4}-\d{2}-\d{2}$/) + .optional(), + children: z.array(pageBlock).max(100).optional(), + }), +}; + +/** The native Zod schema of each speed case's JSON Schema. */ +export const nativeCases: ReadonlyMap = new Map( + cases.map((c) => { + const schema = native[c.name]; + if (schema === undefined) throw new Error(`No native Zod schema for the case ${c.name}`); + return [c.schema, schema]; + }), +); + +type Json = Record; + +/** + * The native Zod schema a developer would write for a corpus schema (corpus.ts): objects (closed ones strict), + * strings with bounds, patterns and enums, bounded integers, numbers, booleans, string arrays, descriptions, and + * $defs reused through their constant. Throws for any other shape. + */ +export function nativeZod(schema: Json, root: Json = schema): z.ZodType { + if (typeof schema.$ref === "string") { + const name = schema.$ref.replace("#/$defs/", ""); + return nativeZod((root.$defs as Record)[name], root); + } + const built = nativeType(schema, root); + return typeof schema.description === "string" ? built.describe(schema.description) : built; +} + +function nativeType(schema: Json, root: Json): z.ZodType { + switch (schema.type) { + case "object": { + const required = new Set(schema.required as string[]); + const shape: Record = {}; + for (const [key, value] of Object.entries(schema.properties as Record)) { + const property = nativeZod(value, root); + shape[key] = required.has(key) ? property : property.optional(); + } + return schema.additionalProperties === false ? z.strictObject(shape) : z.object(shape); + } + case "string": { + if (Array.isArray(schema.enum)) return z.enum(schema.enum as [string, ...string[]]); + let string = z.string(); + if (typeof schema.minLength === "number") string = string.min(schema.minLength); + if (typeof schema.maxLength === "number") string = string.max(schema.maxLength); + if (typeof schema.pattern === "string") string = string.regex(new RegExp(schema.pattern)); + return string; + } + case "integer": + return z + .number() + .int() + .min(schema.minimum as number) + .max(schema.maximum as number); + case "number": + return z.number(); + case "boolean": + return z.boolean(); + case "array": + return z.array(z.string()).max(schema.maxItems as number); + default: + throw new Error(`No native Zod builder for ${JSON.stringify(schema).slice(0, 80)}`); + } +} + +/** z.fromJSONSchema with the draft given; it accepts only local $refs, so remote documents are not passed. */ +export function zodFromJsonSchema(schema: JsonSchema, draft: "2020-12" | "7"): z.ZodType { + return z.fromJSONSchema(schema as never, { defaultTarget: draft === "7" ? "draft-7" : "draft-2020-12" }); +}