diff --git a/package-lock.json b/package-lock.json index 206abbcbb..35a0c92eb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1515,6 +1515,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@silvia-odwyer/photon": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@silvia-odwyer/photon/-/photon-0.3.3.tgz", + "integrity": "sha512-8BhUjEch4slwRe8uXnaA4vcA5uiiOTT90UMsxulOj2gN98X1p0q9Z4Ysk4DkD05uNgbR9XoSqtZ37w+33w4QKQ==", + "license": "Apache-2.0" + }, "node_modules/@silvia-odwyer/photon-node": { "version": "0.3.4", "resolved": "https://registry.npmjs.org/@silvia-odwyer/photon-node/-/photon-node-0.3.4.tgz", @@ -5280,6 +5286,7 @@ "dependencies": { "@earendil-works/chord": "^1.1.0", "@earendil-works/pi-ai": "^1.1.0", + "@silvia-odwyer/photon": "0.3.3", "diff": "8.0.4", "typebox": "1.3.27" }, diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 52cd1f3b2..ed150b865 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -18,6 +18,7 @@ - `ToolRegistration.callers` (`model`, `tools`, or both, the default) and `pi.agent` `modelTools`: who may call a tool, and which of its tools a conversation offers the model; `Agent.callable` lists the tools nested calls resolve among. `addTools` also extends `modelTools`. - `TaskOptions.abandonOnRestart`: a task its creator awaits only in memory is aborted, with its owned work, when a later Harness starts scheduling, before any of it runs again. Its mark carries `abortReason: "restart"`, which cascades; such a task waits for a missing definition instead of becoming `orphaned`. Nested calls and `api.createTask()` children of tools that are not replay-safe get it by default. No task below an owner with cancellation intent starts a run phase any more, even before the cascade marks it. - `TaskRuntime.abortOwned(id, context)`: abort a task this task owns and wait until it is terminal. +- `read` returns images as one image block instead of an `unsupported_image` error: PNG, JPEG, GIF, and WebP within the model's image limits (by default 2000x2000 pixels and 4.5 MB of base64) as they are, or, with `createCodingTools({ images })`, oriented, converted, and shrunk to fit. `@earendil-works/pi-durable/images` provides a Photon (WebAssembly) image processor, with `/images/node` and `/images/cloudflare` loaders; nothing loads it unless imported. A model without image input gets a diagnostic saying it sees a placeholder. ### Fixed diff --git a/packages/durable/README.md b/packages/durable/README.md index 765fea34a..9bb8629d0 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -147,7 +147,18 @@ An extension may bring `tools`, `sections`, `hooks`, `wraps` (decorators of a to ## Tools -`@earendil-works/pi-durable/tools` provides `read`, `write`, `edit`, and `bash`, and the `CodingTools` extension with all four. They touch files and processes only through the call's environment (see [Environment](#environment)). Reading images is not supported yet. +`@earendil-works/pi-durable/tools` provides `read`, `write`, `edit`, and `bash`, and the `CodingTools` extension with all four. They touch files and processes only through the call's environment (see [Environment](#environment)). + +`read` returns an image file as one image block, which programs calling it get as an `ImageContent`. Limits are the model's `inputLimits.images.resize`, by default 2000x2000 pixels and 4.5 MB of base64. Without an image processor it sends PNG, JPEG, GIF, and WebP files within them (dimensions read from the file's header) as they are, without decoding them, and refuses larger ones and other formats. With one, it decodes every image, turns it upright by its EXIF orientation, converts BMP to PNG, and shrinks it to fit. The Photon-based processor runs on WebAssembly and is only loaded when you import it: + +```typescript +import { createNodePhotonImages } from "@earendil-works/pi-durable/images/node"; // or /images/cloudflare, or createPhotonImages(wasm) from /images +import { createCodingTools } from "@earendil-works/pi-durable/tools"; + +registry.install(createCodingTools({ images: await createNodePhotonImages() })); +``` + +A model without image input sees a placeholder instead of the image, and `read` says so in a diagnostic. The Photon processor works on the calling thread, about a second and a half for a 12-megapixel photo, and decoding takes the image's full size in WebAssembly memory, which does not shrink again; leave it out on memory-constrained hosts such as Cloudflare Workers unless images stay small. Define your own tool with a TypeBox schema. `defineTool()` types `args` from `parameters`, which the Harness validates before `execute()`. `api.output()` streams running output, which becomes the result when `execute()` returns no `output`: diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index 18d42868e..3f6a1652c 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -3294,12 +3294,35 @@ and the error text as its message. Their content is the durable partial output, if any, and `details` is the tool's last reported value, if any. `@earendil-works/pi-durable/tools` provides `read`, `write`, `edit`, and `bash` -factories, ported from the agent harness tools, and the `CodingTools` extension -with all four. They use only `api.env`; nothing -installs them automatically. `read` does not return images yet. It reads a -file through `openBinaryReader`: image detection reads the header (and a PNG's -chunk headers), `scanLines` counts and locates the selected lines, and only the -shown head is read and decoded, so its cost and transfer are bounded by the +factories, ported from the agent harness tools, the `CodingTools` extension +with all four, and `createCodingTools({ images })`, the same with an image +processor for `read`. They use only `api.env`; nothing installs them +automatically. + +`read` returns an image file, detected by content, as one image block with an +`info` diagnostic (code `image`) saying its type and what was done to it, so +programs get a bare `ImageContent`. An image must be read whole: unlike text, +one that grows while it is read is read again. The limits are the +conversation model's `inputLimits.images.resize`, each absent one +`DEFAULT_IMAGE_LIMITS` (2000x2000 pixels, 4.5 MiB of base64). Without an +`ImageProcessor`, `read` passes PNG, JPEG, GIF, and WebP through, undecoded, +when the file size puts their base64 within the byte limit (checked before +reading the file) and their header declares dimensions within the limits; +otherwise, and for BMP, it returns an error result (code `unsupported_image`). +With one, `await processor.prepare(bytes, mimeType, limits)` returns the image fitted to the limits, or `undefined` for an error result; the +diagnostic says when its format changed or it was resized, with the factor that +maps coordinates back. The Photon processor decodes every image, returns it as +it is when it is upright, inline, and within the limits, and otherwise +re-encodes it as PNG or JPEG (JPEG first for JPEG sources). `@earendil-works/pi-durable/images` provides `createPhotonImages(wasm)`, +on Photon's WebAssembly, given its module or bytes; `/images/node` reads the +wasm from the installed package and `/images/cloudflare` imports it as a +Workers module. Neither is loaded unless imported. A conversation whose model +does not take images still gets the image; pi-ai replaces it with a placeholder +in requests, and the diagnostic says so. + +For text, `read` reads a file through `openBinaryReader`: image detection reads +the header (and a PNG's chunk headers), `scanLines` counts and locates the +selected lines, and only the shown head is read and decoded, so its cost and transfer are bounded by the output limits plus one pass over the file inside the environment. Its result is exactly that of decoding the whole file, splitting it into lines, and truncating the selection. A file that changes while it is read is read again diff --git a/packages/durable/package.json b/packages/durable/package.json index a26b29117..14903974d 100644 --- a/packages/durable/package.json +++ b/packages/durable/package.json @@ -30,6 +30,24 @@ "import": "./dist/tools/index.js", "default": "./dist/tools/index.js" }, + "./images": { + "source": "./src/images/index.ts", + "types": "./dist/images/index.d.ts", + "import": "./dist/images/index.js", + "default": "./dist/images/index.js" + }, + "./images/node": { + "source": "./src/images/node.ts", + "types": "./dist/images/node.d.ts", + "import": "./dist/images/node.js", + "default": "./dist/images/node.js" + }, + "./images/cloudflare": { + "source": "./src/images/cloudflare.ts", + "types": "./dist/images/cloudflare.d.ts", + "import": "./dist/images/cloudflare.js", + "default": "./dist/images/cloudflare.js" + }, "./storage/memory": { "source": "./src/storage/memory.ts", "types": "./dist/storage/memory.d.ts", @@ -110,6 +128,7 @@ "dependencies": { "@earendil-works/chord": "^1.1.0", "@earendil-works/pi-ai": "^1.1.0", + "@silvia-odwyer/photon": "0.3.3", "diff": "8.0.4", "typebox": "1.3.27" }, diff --git a/packages/durable/src/images/cloudflare.ts b/packages/durable/src/images/cloudflare.ts new file mode 100644 index 000000000..96c74a394 --- /dev/null +++ b/packages/durable/src/images/cloudflare.ts @@ -0,0 +1,10 @@ +// Wrangler's default `CompiledWasm` rule turns this import into a `WebAssembly.Module`; Workers cannot compile +// WebAssembly from bytes at runtime. Other bundlers do not support it: use `createPhotonImages()` there. +import photonWasm from "@silvia-odwyer/photon/photon_rs_bg.wasm"; +import type { ImageProcessor } from "../tools/image-processor.ts"; +import { createPhotonImages } from "./index.ts"; + +/** `createPhotonImages()` for Cloudflare Workers and Durable Objects, built with Wrangler. */ +export function createCloudflarePhotonImages(): ImageProcessor { + return createPhotonImages(photonWasm); +} diff --git a/packages/durable/src/images/exif.ts b/packages/durable/src/images/exif.ts new file mode 100644 index 000000000..3dcd278f0 --- /dev/null +++ b/packages/durable/src/images/exif.ts @@ -0,0 +1,70 @@ +// EXIF orientation of JPEG and WebP files, read from the bytes without decoding the image. + +/** The EXIF orientation of a JPEG or WebP, 1 to 8; 1 when absent or unreadable. */ +export function exifOrientation(bytes: Uint8Array): number { + const tiff = tiffStart(bytes); + if (tiff < 0 || tiff + 8 > bytes.length) return 1; + const little = bytes[tiff] === 0x49 && bytes[tiff + 1] === 0x49; + const u16 = (at: number): number => + little ? bytes[at]! | (bytes[at + 1]! << 8) : (bytes[at]! << 8) | bytes[at + 1]!; + const u32 = (at: number): number => + little + ? (bytes[at]! | (bytes[at + 1]! << 8) | (bytes[at + 2]! << 16) | (bytes[at + 3]! << 24)) >>> 0 + : ((bytes[at]! << 24) | (bytes[at + 1]! << 16) | (bytes[at + 2]! << 8) | bytes[at + 3]!) >>> 0; + const directory = tiff + u32(tiff + 4); + if (directory + 2 > bytes.length) return 1; + const entries = u16(directory); + for (let index = 0; index < entries; index++) { + const entry = directory + 2 + index * 12; + if (entry + 12 > bytes.length) return 1; + // Tag 0x0112 is Orientation, a SHORT whose value sits in the entry. + if (u16(entry) === 0x0112) { + const value = u16(entry + 8); + return value >= 1 && value <= 8 ? value : 1; + } + } + return 1; +} + +/** Where the TIFF header of a JPEG's APP1 or a WebP's EXIF chunk starts; -1 when there is none. */ +function tiffStart(bytes: Uint8Array): number { + if (bytes[0] === 0xff && bytes[1] === 0xd8) { + for (let offset = 2; offset + 4 <= bytes.length; ) { + if (bytes[offset] !== 0xff) return -1; + const marker = bytes[offset + 1]!; + if (marker === 0xff) { + offset++; + continue; + } + if (marker === 0xe1 && isExifHeader(bytes, offset + 4)) return offset + 10; + offset += 2 + ((bytes[offset + 2]! << 8) | bytes[offset + 3]!); + } + return -1; + } + if (ascii(bytes, 0, "RIFF") && ascii(bytes, 8, "WEBP")) { + for (let offset = 12; offset + 8 <= bytes.length; ) { + const size = + (bytes[offset + 4]! | + (bytes[offset + 5]! << 8) | + (bytes[offset + 6]! << 16) | + (bytes[offset + 7]! << 24)) >>> + 0; + // Some WebP files prefix the TIFF header with "Exif\0\0". + if (ascii(bytes, offset, "EXIF")) return isExifHeader(bytes, offset + 8) ? offset + 14 : offset + 8; + // RIFF chunks are padded to an even size. + offset += 8 + size + (size % 2); + } + } + return -1; +} + +function isExifHeader(bytes: Uint8Array, offset: number): boolean { + return ascii(bytes, offset, "Exif") && bytes[offset + 4] === 0 && bytes[offset + 5] === 0; +} + +function ascii(bytes: Uint8Array, offset: number, text: string): boolean { + for (let index = 0; index < text.length; index++) { + if (bytes[offset + index] !== text.charCodeAt(index)) return false; + } + return true; +} diff --git a/packages/durable/src/images/index.ts b/packages/durable/src/images/index.ts new file mode 100644 index 000000000..ec9a2e0a3 --- /dev/null +++ b/packages/durable/src/images/index.ts @@ -0,0 +1,140 @@ +// The package declares only `module`, which bundlers use but Node does not: import the file itself. +import { + fliph, + flipv, + initSync, + PhotonImage, + resize, + SamplingFilter, + type SyncInitInput, +} from "@silvia-odwyer/photon/photon_rs.js"; +import type { ImageLimits, ImageProcessor, PreparedImage } from "../tools/image-processor.ts"; +import { base64Length, INLINE_IMAGE_TYPES, toBase64 } from "../tools/image-processor.ts"; +import { exifOrientation } from "./exif.ts"; + +/** JPEG qualities tried in order. */ +const JPEG_QUALITIES = [80, 70, 55, 40]; + +/** + * An `ImageProcessor` on Photon (Rust compiled to WebAssembly), for `createCodingTools({ images })`. `wasm` is + * Photon's `photon_rs_bg.wasm` as bytes or as a compiled module: `@earendil-works/pi-durable/images/node` reads it from + * the installed package, `@earendil-works/pi-durable/images/cloudflare` imports it as a Workers module. Elsewhere, such + * as a browser, fetch the file and pass its bytes. Photon loads once per process; later calls reuse the first `wasm`. + * + * An image already upright, in a format providers take inline, and within the limits is returned as it is, once it + * decodes. Otherwise it is turned upright by its EXIF orientation, scaled to the + * maximum width and height, and encoded as PNG or JPEG at falling qualities (JPEG first for JPEG sources), then at + * smaller dimensions, 3/4 at a time, until the base64 fits. Formats providers do not take inline (BMP) become PNG. + * + * The work runs on the calling thread: about a second and a half for a 12-megapixel photo. Decoding takes the image's + * RGBA size in WebAssembly memory, which does not shrink again; leave the processor out on memory-constrained hosts. + */ +export function createPhotonImages(wasm: SyncInitInput): ImageProcessor { + initSync({ module: wasm }); + return { prepare: async (bytes, mimeType, limits) => prepare(bytes, mimeType, limits) }; +} + +function prepare(bytes: Uint8Array, mimeType: string, limits: ImageLimits): PreparedImage | undefined { + const orientation = exifOrientation(bytes); + const inline = INLINE_IMAGE_TYPES.has(mimeType); + let image: PhotonImage | undefined; + try { + // Decoded even when it is returned as it is: a damaged file is an error, not an image the provider rejects. + image = PhotonImage.new_from_byteslice(bytes); + if ( + inline && + orientation === 1 && + image.get_width() <= limits.maxWidth && + image.get_height() <= limits.maxHeight && + base64Length(bytes.byteLength) <= limits.maxBytes + ) { + return { data: toBase64(bytes), mimeType }; + } + if (orientation >= 5) { + const upright = transposed(image, orientation); + // Freed first, so the upright copy reuses its WebAssembly memory. + image.free(); + image = undefined; + image = new PhotonImage(upright.pixels, upright.width, upright.height); + } else if (orientation !== 1) { + if (orientation !== 4) fliph(image); + if (orientation !== 2) flipv(image); + } + const width = image.get_width(); + const height = image.get_height(); + const scale = Math.min(1, limits.maxWidth / width, limits.maxHeight / height); + let target = { width: Math.max(1, Math.round(width * scale)), height: Math.max(1, Math.round(height * scale)) }; + for (;;) { + const encoded = encodeWithin(image, target, limits.maxBytes, mimeType === "image/jpeg"); + if (encoded !== undefined) { + const resized = + target.width === width && target.height === height + ? {} + : { resized: { from: { width, height }, to: target } }; + const converted = encoded.mimeType === mimeType ? {} : { convertedFrom: mimeType }; + return { ...encoded, ...converted, ...resized }; + } + if (target.width === 1 && target.height === 1) return undefined; + target = { + width: Math.max(1, Math.floor(target.width * 0.75)), + height: Math.max(1, Math.floor(target.height * 0.75)), + }; + } + } catch { + // Photon throws for bytes it cannot decode. + return undefined; + } finally { + image?.free(); + } +} + +/** + * `image` at `size` in the first encoding whose base64 fits `maxBytes`: PNG, then JPEG at falling qualities, or the + * JPEG qualities first when `jpegFirst`; `undefined` if none fits. + */ +function encodeWithin( + image: PhotonImage, + size: { readonly width: number; readonly height: number }, + maxBytes: number, + jpegFirst: boolean, +): { readonly data: string; readonly mimeType: string } | undefined { + const same = size.width === image.get_width() && size.height === image.get_height(); + const scaled = same ? image : resize(image, size.width, size.height, SamplingFilter.Lanczos3); + try { + const png = { mimeType: "image/png", encode: () => scaled.get_bytes() }; + const jpegs = JPEG_QUALITIES.map((quality) => ({ + mimeType: "image/jpeg", + encode: () => scaled.get_bytes_jpeg(quality), + })); + for (const candidate of jpegFirst ? [...jpegs, png] : [png, ...jpegs]) { + const bytes = candidate.encode(); + if (base64Length(bytes.byteLength) <= maxBytes) return { data: toBase64(bytes), mimeType: candidate.mimeType }; + } + return undefined; + } finally { + if (!same) scaled.free(); + } +} + +/** + * The RGBA pixels of `image` turned upright for EXIF `orientation` 5 to 8, the orientations that swap width and + * height. Photon's own `rotate` drops pixels at right angles, so the pixels are moved here. + */ +function transposed( + image: PhotonImage, + orientation: number, +): { readonly pixels: Uint8Array; readonly width: number; readonly height: number } { + const width = image.get_width(); + const height = image.get_height(); + const source = image.get_raw_pixels(); + const pixels = new Uint8Array(source.length); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + // Where (x, y) lands in the upright image, which is `height` wide. + const outX = orientation === 5 || orientation === 8 ? y : height - 1 - y; + const outY = orientation === 5 || orientation === 6 ? x : width - 1 - x; + pixels.set(source.subarray((y * width + x) * 4, (y * width + x) * 4 + 4), (outY * height + outX) * 4); + } + } + return { pixels, width: height, height: width }; +} diff --git a/packages/durable/src/images/node.ts b/packages/durable/src/images/node.ts new file mode 100644 index 000000000..02aa474f4 --- /dev/null +++ b/packages/durable/src/images/node.ts @@ -0,0 +1,13 @@ +import { readFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import type { ImageProcessor } from "../tools/image-processor.ts"; +import { createPhotonImages } from "./index.ts"; + +/** + * `createPhotonImages()` with Photon's WebAssembly read from the installed `@silvia-odwyer/photon`, for Node and Bun. + * A single-file binary cannot resolve it: read its own copy of `photon_rs_bg.wasm` and call `createPhotonImages()`. + */ +export async function createNodePhotonImages(): Promise { + const path = createRequire(import.meta.url).resolve("@silvia-odwyer/photon/photon_rs_bg.wasm"); + return createPhotonImages(await readFile(path)); +} diff --git a/packages/durable/src/images/photon-wasm.d.ts b/packages/durable/src/images/photon-wasm.d.ts new file mode 100644 index 000000000..749614c14 --- /dev/null +++ b/packages/durable/src/images/photon-wasm.d.ts @@ -0,0 +1,5 @@ +// Imported by cloudflare.ts. Wrangler's default `CompiledWasm` rule makes a .wasm import a compiled module. +declare module "@silvia-odwyer/photon/photon_rs_bg.wasm" { + const module: WebAssembly.Module; + export default module; +} diff --git a/packages/durable/src/tools/image-processor.ts b/packages/durable/src/tools/image-processor.ts new file mode 100644 index 000000000..9781dffe7 --- /dev/null +++ b/packages/durable/src/tools/image-processor.ts @@ -0,0 +1,69 @@ +/** Limits an image the model sees must keep; the defaults below are the coding agent's. */ +export type ImageLimits = { + /** Pixels. Default 2000. */ + readonly maxWidth: number; + /** Pixels. Default 2000. */ + readonly maxHeight: number; + /** Bytes of base64. Default 4.5 MiB, below Anthropic's 5 MB. */ + readonly maxBytes: number; +}; + +export const DEFAULT_IMAGE_LIMITS: ImageLimits = { maxWidth: 2000, maxHeight: 2000, maxBytes: 4.5 * 1024 * 1024 }; + +/** An image the model can take: a supported format, within the limits, as base64. */ +export type PreparedImage = { + readonly data: string; + readonly mimeType: string; + /** Set when the image was resized: its size before and after, for the model to map coordinates back. */ + readonly resized?: { + readonly from: { readonly width: number; readonly height: number }; + readonly to: { readonly width: number; readonly height: number }; + }; + /** The original format, set when it differs from `mimeType`: BMP becomes PNG, a large PNG may become JPEG. */ + readonly convertedFrom?: string; +}; + +/** + * Decodes, orients, resizes, and re-encodes images so the model can take them. `read` uses one when given; without + * one, it passes supported images within the byte limit through as they are. `@earendil-works/pi-durable/images` + * provides a Photon-based one. + */ +export type ImageProcessor = { + /** + * The image, prepared to fit `limits`, or `undefined` when it cannot be decoded or made to fit. A promise, so a + * processor can do the work off the event loop. + */ + prepare(bytes: Uint8Array, mimeType: string, limits: ImageLimits): Promise; +}; + +/** Formats every provider takes inline; others, such as BMP, need converting. */ +export const INLINE_IMAGE_TYPES: ReadonlySet = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]); + +const BASE64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +/** Standard base64 of `bytes`, without platform APIs such as `Buffer` or `btoa`. */ +export function toBase64(bytes: Uint8Array): string { + const parts: string[] = []; + let chunk = ""; + for (let i = 0; i < bytes.length; i += 3) { + const a = bytes[i]!; + const b = bytes[i + 1]; + const c = bytes[i + 2]; + chunk += BASE64[a >> 2]!; + chunk += BASE64[((a & 3) << 4) | ((b ?? 0) >> 4)]!; + chunk += b === undefined ? "=" : BASE64[((b & 15) << 2) | ((c ?? 0) >> 6)]!; + chunk += c === undefined ? "=" : BASE64[c & 63]!; + // Join in pieces: one string grown a character at a time is slow for megabytes. + if (chunk.length >= 8192) { + parts.push(chunk); + chunk = ""; + } + } + parts.push(chunk); + return parts.join(""); +} + +/** The length of the base64 of `byteLength` bytes. */ +export function base64Length(byteLength: number): number { + return Math.ceil(byteLength / 3) * 4; +} diff --git a/packages/durable/src/tools/image.ts b/packages/durable/src/tools/image.ts index dbb2f2a3b..062a10d35 100644 --- a/packages/durable/src/tools/image.ts +++ b/packages/durable/src/tools/image.ts @@ -72,6 +72,52 @@ function isAnimatedPng(buffer: Uint8Array): boolean { return false; } +/** + * The width and height an image of `mimeType` declares in its header, without decoding it: PNG's IHDR, GIF's screen, + * WebP's VP8, VP8L, or VP8X chunk, JPEG's first SOF segment. `undefined` when the header cannot be read. + */ +export function imageDimensions( + bytes: Uint8Array, + mimeType: string, +): { readonly width: number; readonly height: number } | undefined { + switch (mimeType) { + case "image/png": + return bytes.length < 24 ? undefined : { width: readUint32BE(bytes, 16), height: readUint32BE(bytes, 20) }; + case "image/gif": + return bytes.length < 10 ? undefined : { width: readUint16LE(bytes, 6), height: readUint16LE(bytes, 8) }; + case "image/webp": + if (bytes.length < 30) return undefined; + if (startsWithAscii(bytes, 12, "VP8 ")) + return { width: readUint16LE(bytes, 26) & 0x3fff, height: readUint16LE(bytes, 28) & 0x3fff }; + if (startsWithAscii(bytes, 12, "VP8L")) { + const bits = readUint32LE(bytes, 21); + return { width: (bits & 0x3fff) + 1, height: ((bits >>> 14) & 0x3fff) + 1 }; + } + if (startsWithAscii(bytes, 12, "VP8X")) { + const uint24 = (offset: number): number => readUint16LE(bytes, offset) + ((bytes[offset + 2] ?? 0) << 16); + return { width: uint24(24) + 1, height: uint24(27) + 1 }; + } + return undefined; + case "image/jpeg": + for (let offset = 2; offset + 9 <= bytes.length; ) { + if (bytes[offset] !== 0xff) return undefined; + const marker = bytes[offset + 1]!; + if (marker === 0xff) { + offset++; + continue; + } + // SOF0 to SOF15, except DHT (C4), JPG (C8), and DAC (CC), which share the range. + if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) { + return { width: readUint16BE(bytes, offset + 7), height: readUint16BE(bytes, offset + 5) }; + } + offset += 2 + readUint16BE(bytes, offset + 2); + } + return undefined; + default: + return undefined; + } +} + function isBmp(buffer: Uint8Array): boolean { if (buffer.length < 26) return false; const declaredFileSize = readUint32LE(buffer, 2); @@ -100,6 +146,10 @@ function readUint16LE(buffer: Uint8Array, offset: number): number { return (buffer[offset] ?? 0) + ((buffer[offset + 1] ?? 0) << 8); } +function readUint16BE(buffer: Uint8Array, offset: number): number { + return ((buffer[offset] ?? 0) << 8) + (buffer[offset + 1] ?? 0); +} + function readUint32BE(buffer: Uint8Array, offset: number): number { return ( (buffer[offset] ?? 0) * 0x1000000 + diff --git a/packages/durable/src/tools/index.ts b/packages/durable/src/tools/index.ts index b277dada7..adf06d188 100644 --- a/packages/durable/src/tools/index.ts +++ b/packages/durable/src/tools/index.ts @@ -1,7 +1,7 @@ import { defineExtension } from "../harness/define.ts"; import { createBashTool } from "./bash.ts"; import { createEditTool } from "./edit.ts"; -import { createReadTool } from "./read.ts"; +import { createReadTool, type ReadToolOptions } from "./read.ts"; import { createWriteTool } from "./write.ts"; export { @@ -15,11 +15,25 @@ export { type PowerShellToolOptions, } from "./bash.ts"; export { createEditTool, type EditToolDetails, type EditToolInput } from "./edit.ts"; -export { createReadTool, type ReadToolDetails, type ReadToolInput } from "./read.ts"; +export { + DEFAULT_IMAGE_LIMITS, + type ImageLimits, + type ImageProcessor, + type PreparedImage, +} from "./image-processor.ts"; +export { createReadTool, type ReadToolDetails, type ReadToolInput, type ReadToolOptions } from "./read.ts"; export { createWriteTool, type WriteToolInput } from "./write.ts"; -/** `read`, `write`, `edit`, and `bash`; nothing installs it automatically. `createPowerShellTool()` adds `powershell`. */ -export const CodingTools = defineExtension({ - name: "coding-tools", - tools: [createReadTool(), createWriteTool(), createEditTool(), createBashTool()], -}); +/** + * The `coding-tools` extension: `read`, `write`, `edit`, and `bash`; nothing installs it automatically. + * `createPowerShellTool()` adds `powershell`. `images` prepares the images `read` returns (see `ReadToolOptions`). + */ +export function createCodingTools(options: ReadToolOptions = {}) { + return defineExtension({ + name: "coding-tools", + tools: [createReadTool(options), createWriteTool(), createEditTool(), createBashTool()], + }); +} + +/** `createCodingTools()` without an image processor. */ +export const CodingTools = createCodingTools(); diff --git a/packages/durable/src/tools/read.ts b/packages/durable/src/tools/read.ts index 1b353c8f4..bea5c812c 100644 --- a/packages/durable/src/tools/read.ts +++ b/packages/durable/src/tools/read.ts @@ -10,7 +10,7 @@ import { } from "../env/index.ts"; import { defineTool } from "../harness/define.ts"; import { characterEnd } from "../harness/output.ts"; -import type { ToolDiagnostic, ToolRegistration } from "../harness/types.ts"; +import type { ToolDiagnostic, ToolExecutionApi, ToolExecutionResult, ToolRegistration } from "../harness/types.ts"; import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, @@ -20,7 +20,15 @@ import { utf8ByteLength, } from "../truncate.ts"; import { requireEnv } from "./env.ts"; -import { detectSupportedImageMimeTypeOf } from "./image.ts"; +import { detectSupportedImageMimeTypeOf, imageDimensions } from "./image.ts"; +import { + base64Length, + DEFAULT_IMAGE_LIMITS, + type ImageLimits, + type ImageProcessor, + INLINE_IMAGE_TYPES, + toBase64, +} from "./image-processor.ts"; import { resolveReadToolPath } from "./path-utils.ts"; const readSchema = Type.Object({ @@ -69,11 +77,20 @@ async function readHead( return text + decoder.decode(); } -/** Reads text files. Remarks about truncation and continuation are diagnostics; the content is only file text. */ -export function createReadTool(): ToolRegistration { +export type ReadToolOptions = { + /** Resizes and converts images; without one, images within `DEFAULT_IMAGE_LIMITS.maxBytes` pass through as they are. */ + readonly images?: ImageProcessor; +}; + +/** + * Reads text files and images. Remarks about truncation, continuation, and image processing are diagnostics; the content + * is only file text, or one image, which programs get as its `ImageContent`. + */ +export function createReadTool(options: ReadToolOptions = {}): ToolRegistration { + const formats = options.images === undefined ? "jpg, png, gif, webp" : "jpg, png, gif, webp, bmp"; return defineTool({ name: "read", - description: `Read the contents of a text file. Output is truncated to ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.`, + description: `Read the contents of a file. Supports text files and images (${formats}). Images are sent as attachments. For text files, output is truncated to ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.`, parameters: readSchema, async execute(args, api, context) { const { path, offset, limit } = args; @@ -82,14 +99,28 @@ export function createReadTool(): ToolRegistration getOrThrow(await reader.read(position, length, context)), + }); + const result = + mimeType === undefined + ? await readText(reader, path, offset, limit, context) + : await readImage( + reader, + before, + path, + mimeType, + options.images, + await imageModel(api, context), + context, + ); const after = getOrThrow(await reader.info(context)); - if (after.size > before.size || (after.size === before.size && after.mtimeMs === before.mtimeMs)) { - return result; - } + const unchanged = after.size === before.size && after.mtimeMs === before.mtimeMs; + if (unchanged || (mimeType === undefined && after.size > before.size)) return result; if (attempt === 1) throw new Error(`${path} changed while it was read`); } } finally { @@ -99,37 +130,102 @@ export function createReadTool(): ToolRegistration> { + let image: { readonly data: string; readonly mimeType: string }; + const notes: string[] = []; + if (processor !== undefined) { + const bytes = getOrThrow(await reader.read(0, info.size, context)); + const prepared = await processor.prepare(bytes, mimeType, limits); + if (prepared === undefined) + return imageError(`${path} is an image (${mimeType}) that cannot be prepared for the model`); + image = prepared; + if (prepared.convertedFrom !== undefined) + notes.push(`Converted from ${prepared.convertedFrom} to ${prepared.mimeType}.`); + if (prepared.resized !== undefined) { + const { from, to } = prepared.resized; + const scale = (from.width / to.width).toFixed(2); + notes.push( + `Resized from ${from.width}x${from.height} to ${to.width}x${to.height}. Multiply coordinates by ${scale} to map them to the original.`, + ); + } + } else { + if (!INLINE_IMAGE_TYPES.has(mimeType)) { + return imageError( + `${path} is an image (${mimeType}) that needs converting, and no image processor is configured`, + ); + } + const unshrinkable = "and no image processor is configured to shrink it"; + if (base64Length(info.size) > limits.maxBytes) { + return imageError( + `${path} is an image (${mimeType}) of ${formatSize(info.size)}, too large to send, ${unshrinkable}`, + ); + } + const bytes = getOrThrow(await reader.read(0, info.size, context)); + const size = imageDimensions(bytes, mimeType); + if (size === undefined) return imageError(`${path} is an image (${mimeType}) whose size cannot be read`); + if (size.width > limits.maxWidth || size.height > limits.maxHeight) { + return imageError( + `${path} is an image (${mimeType}) of ${size.width}x${size.height}, larger than ${limits.maxWidth}x${limits.maxHeight}, ${unshrinkable}`, + ); + } + image = { data: toBase64(bytes), mimeType }; + } + if (!vision) notes.push("The current model does not support images; it sees a placeholder instead."); + const diagnostics: ToolDiagnostic[] = [ + { severity: "info", code: "image", message: [`Read image file [${image.mimeType}].`, ...notes].join(" ") }, + ]; + return { output: [{ type: "image", data: image.data, mimeType: image.mimeType }], diagnostics }; +} + +/** What the conversation's model takes: whether images, and within which limits. */ +type ImageModel = { readonly vision: boolean; readonly limits: ImageLimits }; + +/** + * The conversation model's image input: `DEFAULT_IMAGE_LIMITS` overridden by its `inputLimits.images.resize`. Without + * a resolvable model, it counts as one that takes images within the defaults. + */ +async function imageModel(api: ToolExecutionApi, context: Context): Promise { + const ref = (await api.agent(context)).model; + const model = ref === undefined ? undefined : api.models.getModel(ref.provider, ref.modelId); + const resize = model?.inputLimits?.images?.resize; + return { + vision: model?.input.includes("image") ?? true, + limits: { + maxWidth: resize?.maxWidth ?? DEFAULT_IMAGE_LIMITS.maxWidth, + maxHeight: resize?.maxHeight ?? DEFAULT_IMAGE_LIMITS.maxHeight, + maxBytes: resize?.maxBytes ?? DEFAULT_IMAGE_LIMITS.maxBytes, + }, + }; +} + +function imageError(message: string): ToolExecutionResult { + return { output: [], isError: true, diagnostics: [{ severity: "error", code: "unsupported_image", message }] }; +} + /** * The read result for the opened file. It equals decoding the whole file with `TextDecoder`, splitting it on `\n`, * and bounding the selected lines with `truncateHead`, while reading only one scan's worth of the file plus the head. */ async function readText( reader: BinaryReader, - info: FileInfo, path: string, offset: number | undefined, limit: number | undefined, context: Context, ) { - const mimeType = await detectSupportedImageMimeTypeOf({ - size: info.size, - read: async (position, length) => getOrThrow(await reader.read(position, length, context)), - }); - if (mimeType) { - // Image content is not supported yet. - return { - output: [], - isError: true, - diagnostics: [ - { - severity: "error" as const, - code: "unsupported_image", - message: `${path} is an image (${mimeType}); reading images is not supported`, - }, - ], - }; - } - const startLine = offset ? Math.max(0, offset - 1) : 0; const startLineDisplay = startLine + 1; // Lines are selected like `allLines.slice(startLine, endLine)`, which truncates fractional indices. diff --git a/packages/durable/test/images-wasm-module.test.ts b/packages/durable/test/images-wasm-module.test.ts new file mode 100644 index 000000000..9e8111f78 --- /dev/null +++ b/packages/durable/test/images-wasm-module.test.ts @@ -0,0 +1,17 @@ +// Its own file, so Photon is not yet initialized: vitest gives each file fresh modules, and Photon loads only once. +import { readFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { createPhotonImages } from "@earendil-works/pi-durable/images"; +import { DEFAULT_IMAGE_LIMITS } from "@earendil-works/pi-durable/tools"; +import { expect, it } from "vitest"; + +it("createPhotonImages takes a compiled WebAssembly module, as Cloudflare Workers pass it", async () => { + const path = createRequire(import.meta.url).resolve("@silvia-odwyer/photon/photon_rs_bg.wasm"); + const { WebAssembly } = globalThis as unknown as { WebAssembly: { compile(bytes: Uint8Array): Promise } }; + const module = (await WebAssembly.compile(await readFile(path))) as Parameters[0]; + const images = createPhotonImages(module); + // A 1x1 GIF; returned as it is only once it decodes, which fails without an initialized Photon. + const data = "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"; + const prepared = await images.prepare(Buffer.from(data, "base64"), "image/gif", DEFAULT_IMAGE_LIMITS); + expect(prepared).toEqual({ data, mimeType: "image/gif" }); +}); diff --git a/packages/durable/test/tools-images.test.ts b/packages/durable/test/tools-images.test.ts new file mode 100644 index 000000000..fc0c2dcb6 --- /dev/null +++ b/packages/durable/test/tools-images.test.ts @@ -0,0 +1,488 @@ +import { appendFileSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; +import type { Model } from "@earendil-works/pi-ai"; +import type { ToolDiagnostic, ToolExecutionApi, ToolExecutionResult } from "@earendil-works/pi-durable"; +import { createNodePhotonImages } from "@earendil-works/pi-durable/images/node"; +import { createCodingTools } from "@earendil-works/pi-durable/tools"; +import { PhotonImage } from "@silvia-odwyer/photon/photon_rs.js"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { getOrThrow } from "../src/env/index.ts"; +import { NodeExecutionEnv } from "../src/env/node.ts"; +import { exifOrientation } from "../src/images/exif.ts"; +import { imageDimensions } from "../src/tools/image.ts"; +import { DEFAULT_IMAGE_LIMITS, type ImageProcessor, toBase64 } from "../src/tools/image-processor.ts"; +import { createReadTool } from "../src/tools/read.ts"; + +const directory = mkdtempSync(join(tmpdir(), "pi-durable-images-")); +let processor: ImageProcessor; + +beforeAll(async () => { + processor = await createNodePhotonImages(); +}); + +afterAll(() => rmSync(directory, { recursive: true, force: true })); + +type Rgb = readonly [number, number, number]; + +/** A `width` by `height` image of `block`-pixel squares coloured row by row from `colors`. */ +function blocks( + colors: readonly (readonly Rgb[])[], + block: number, +): { rgba: Uint8Array; width: number; height: number } { + const width = colors[0]!.length * block; + const height = colors.length * block; + const rgba = new Uint8Array(width * height * 4); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + rgba.set([...colors[Math.floor(y / block)]![Math.floor(x / block)]!, 255], (y * width + x) * 4); + } + } + return { rgba, width, height }; +} + +/** RGBA pixels of a `width` by `height` image: red, with a green top-left pixel. */ +function pixels(width: number, height: number): Uint8Array { + const rgba = new Uint8Array(width * height * 4); + for (let index = 0; index < width * height; index++) rgba.set([255, 0, 0, 255], index * 4); + rgba.set([0, 255, 0, 255], 0); + return rgba; +} + +function encodeRgba(rgba: Uint8Array, width: number, height: number, format: "png" | "jpeg"): Uint8Array { + const image = new PhotonImage(rgba, width, height); + try { + return format === "png" ? image.get_bytes() : image.get_bytes_jpeg(95); + } finally { + image.free(); + } +} + +function encode(width: number, height: number, format: "png" | "jpeg"): Uint8Array { + return encodeRgba(pixels(width, height), width, height, format); +} + +/** Noise, which neither PNG nor JPEG compresses well, so the byte limit decides. */ +function noisyPng(width: number, height: number): Uint8Array { + const rgba = new Uint8Array(width * height * 4); + let seed = 1; + for (let index = 0; index < rgba.length; index++) { + seed = (seed * 1103515245 + 12345) >>> 0; + rgba[index] = index % 4 === 3 ? 255 : seed >>> 24; + } + return encodeRgba(rgba, width, height, "png"); +} + +/** A 24-bit BMP, which providers do not take inline. */ +function bmp(width: number, height: number): Uint8Array { + const row = Math.ceil((width * 3) / 4) * 4; + const bytes = new Uint8Array(54 + row * height); + const view = new DataView(bytes.buffer); + bytes.set([0x42, 0x4d]); + view.setUint32(2, bytes.length, true); + view.setUint32(10, 54, true); + view.setUint32(14, 40, true); + view.setInt32(18, width, true); + view.setInt32(22, height, true); + view.setUint16(26, 1, true); + view.setUint16(28, 24, true); + view.setUint32(34, row * height, true); + for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) bytes.set([0, 0, 255], 54 + y * row + x * 3); + return bytes; +} + +/** A 1x1 GIF with its logical screen size set to `width` by `height`. */ +function gif(width = 1, height = 1): Uint8Array { + const bytes = Uint8Array.from(Buffer.from("R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7", "base64")); + new DataView(bytes.buffer).setUint16(6, width, true); + new DataView(bytes.buffer).setUint16(8, height, true); + return bytes; +} + +/** A TIFF header whose first directory holds only Orientation = `orientation`. */ +function tiff(orientation: number, littleEndian: boolean): number[] { + const bytes = new Uint8Array(26); + const view = new DataView(bytes.buffer); + bytes.set(littleEndian ? [0x49, 0x49] : [0x4d, 0x4d]); + view.setUint16(2, 42, littleEndian); + view.setUint32(4, 8, littleEndian); + view.setUint16(8, 1, littleEndian); + view.setUint16(10, 0x0112, littleEndian); + view.setUint16(12, 3, littleEndian); + view.setUint32(14, 1, littleEndian); + view.setUint16(18, orientation, littleEndian); + return [...bytes]; +} + +const EXIF_HEADER = [0x45, 0x78, 0x69, 0x66, 0, 0]; + +/** `jpeg` with an APP1 segment whose EXIF says `orientation`, after an APP0 segment when `app0`. */ +function withOrientation(jpeg: Uint8Array, orientation: number, options: { little?: boolean; app0?: boolean } = {}) { + const exif = [...EXIF_HEADER, ...tiff(orientation, options.little ?? false)]; + const app1 = [0xff, 0xe1, (exif.length + 2) >> 8, (exif.length + 2) & 0xff, ...exif]; + const app0 = options.app0 ? [0xff, 0xe0, 0, 16, 0x4a, 0x46, 0x49, 0x46, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0] : []; + return new Uint8Array([...jpeg.subarray(0, 2), ...app0, ...app1, ...jpeg.subarray(2)]); +} + +/** A RIFF WebP with a dummy `VP8 ` chunk and an `EXIF` chunk saying `orientation`, its TIFF data bare or prefixed. */ +function webpWithOrientation(orientation: number, prefixed: boolean): Uint8Array { + const chunk = (type: string, data: number[]): number[] => { + const size = data.length; + const padding = size % 2 === 1 ? [0] : []; + return [...Buffer.from(type), size & 0xff, (size >> 8) & 0xff, 0, 0, ...data, ...padding]; + }; + const exif = [...(prefixed ? EXIF_HEADER : []), ...tiff(orientation, true)]; + const body = [...Buffer.from("WEBP"), ...chunk("VP8 ", [1, 2, 3]), ...chunk("EXIF", exif)]; + return new Uint8Array([...Buffer.from("RIFF"), body.length & 0xff, (body.length >> 8) & 0xff, 0, 0, ...body]); +} + +function decodedImage(data: string): { width: number; height: number; rgba: Uint8Array } { + const image = PhotonImage.new_from_byteslice(Uint8Array.from(Buffer.from(data, "base64"))); + try { + return { width: image.get_width(), height: image.get_height(), rgba: image.get_raw_pixels() }; + } finally { + image.free(); + } +} + +/** + * The environment of a `read` call, with `onRead` seeing each positional read of an opened file, and the largest + * read so far in `reads.largest`. + */ +function readEnv(onRead?: (offset: number, length: number) => void) { + const env = new NodeExecutionEnv({ cwd: directory }); + const reads = { largest: 0 }; + const open = env.openBinaryReader.bind(env); + env.openBinaryReader = async (path, options, ctx) => { + const reader = getOrThrow(await open(path, options, ctx)); + return { + ok: true, + value: { + info: (c) => reader.info(c), + scanLines: (o, c) => reader.scanLines(o, c), + close: (c) => reader.close(c), + read: async (offset, length, c) => { + reads.largest = Math.max(reads.largest, length); + const result = await reader.read(offset, length, c); + onRead?.(offset, length); + return result; + }, + }, + }; + }; + return { env, reads }; +} + +/** Run `read` on `bytes` written to `name`, for a conversation whose model takes `input`. */ +async function read( + name: string, + bytes: Uint8Array, + options: { + readonly images?: ImageProcessor; + readonly input?: readonly string[]; + readonly resize?: { readonly maxWidth?: number; readonly maxHeight?: number; readonly maxBytes?: number }; + readonly env?: NodeExecutionEnv; + readonly tool?: ReturnType; + } = {}, +): Promise { + writeFileSync(join(directory, name), bytes); + const model = { + input: options.input ?? ["text", "image"], + ...(options.resize === undefined ? {} : { inputLimits: { images: { resize: options.resize } } }), + } as unknown as Model; + const api = { + env: options.env ?? new NodeExecutionEnv({ cwd: directory }), + output: () => {}, + diagnostic: () => {}, + details: async () => {}, + agent: async () => ({ model: { provider: "test", modelId: "test" } }), + models: { getModel: () => model }, + } as unknown as ToolExecutionApi; + const tool = options.tool ?? createReadTool(options); + return tool.execute({ path: name }, api, context); +} + +function messages(result: ToolExecutionResult): string[] { + return (result.diagnostics ?? []).map((diagnostic: ToolDiagnostic) => diagnostic.message); +} + +function onlyImage(result: ToolExecutionResult): { data: string; mimeType: string } { + expect(result.output).toHaveLength(1); + return (result.output as { data: string; mimeType: string }[])[0]!; +} + +describe("read of images, without an image processor", () => { + it("returns a supported image within the limits as it is", async () => { + const png = encode(8, 6, "png"); + const result = await read("a.png", png); + expect(result.output).toEqual([{ type: "image", data: toBase64(png), mimeType: "image/png" }]); + expect(messages(result)).toEqual(["Read image file [image/png]."]); + const tiny = await read("a.gif", gif()); + expect(onlyImage(tiny)).toEqual({ type: "image", data: toBase64(gif()), mimeType: "image/gif" }); + }); + + it("refuses an image too large to send without reading it", async () => { + const { env, reads } = readEnv(); + const result = await read("big.png", noisyPng(1100, 1100), { env }); + expect(result).toMatchObject({ output: [], isError: true }); + expect(messages(result)[0]).toMatch(/^big\.png is an image \(image\/png\) of .*, too large to send/); + // Only the header and the chunk walk, in blocks of at most 64 KiB. + expect(reads.largest).toBeLessThanOrEqual(64 * 1024); + }); + + it("refuses an image wider or taller than the limits, by its header", async () => { + const wide = await read("wide.png", encode(2001, 2, "png")); + expect(wide).toMatchObject({ output: [], isError: true }); + expect(messages(wide)).toEqual([ + "wide.png is an image (image/png) of 2001x2, larger than 2000x2000, and no image processor is configured to shrink it", + ]); + const tall = await read("tall.gif", gif(1, 3000)); + expect(messages(tall)[0]).toMatch(/^tall\.gif is an image \(image\/gif\) of 1x3000, larger than 2000x2000/); + }); + + it("refuses a format that needs converting", async () => { + const result = await read("a.bmp", bmp(4, 4)); + expect(result).toMatchObject({ output: [], isError: true }); + expect(messages(result)).toEqual([ + "a.bmp is an image (image/bmp) that needs converting, and no image processor is configured", + ]); + }); + + it("says when the model sees a placeholder instead of the image", async () => { + const result = await read("a.png", encode(4, 4, "png"), { input: ["text"] }); + expect(result.output).toHaveLength(1); + expect(messages(result)).toEqual([ + "Read image file [image/png]. The current model does not support images; it sees a placeholder instead.", + ]); + }); + + it("applies the model's image limits instead of the defaults", async () => { + const png = encode(8, 6, "png"); + const refused = await read("a.png", png, { resize: { maxWidth: 4 } }); + expect(messages(refused)[0]).toMatch(/of 8x6, larger than 4x2000,/); + const resized = await read("a.png", png, { resize: { maxWidth: 4 }, images: processor }); + expect(decodedImage(onlyImage(resized).data)).toMatchObject({ width: 4, height: 3 }); + }); + + it("lists bmp in the description only with a processor", () => { + expect(createReadTool().description).toContain("(jpg, png, gif, webp)"); + expect(createReadTool({ images: processor }).description).toContain("(jpg, png, gif, webp, bmp)"); + }); +}); + +describe("read of an image that changes while it is read", () => { + it("reads it again rather than return a cut-off image, even when it grew", async () => { + const png = encode(8, 6, "png"); + const wholeReads: number[] = []; + const { env } = readEnv((offset, length) => { + // A writer appends during the first read of the whole file; the second read sees it all. + if (offset !== 0 || length < png.length) return; + wholeReads.push(length); + if (wholeReads.length === 1) appendFileSync(join(directory, "growing.png"), new Uint8Array(16)); + }); + const result = await read("growing.png", png, { env }); + expect(wholeReads).toEqual([png.length, png.length + 16]); + const after = [...png, ...new Uint8Array(16)]; + expect(onlyImage(result).data).toBe(toBase64(new Uint8Array(after))); + }); + + it("fails when it changes during the second read too", async () => { + const png = encode(8, 6, "png"); + const { env } = readEnv((offset, length) => { + if (offset === 0 && length >= png.length) appendFileSync(join(directory, "busy.png"), new Uint8Array(1)); + }); + await expect(read("busy.png", png, { env })).rejects.toThrow("busy.png changed while it was read"); + }); +}); + +describe("read of images, with the Photon image processor", () => { + it("passes an image that needs no change through untouched", async () => { + const png = encode(8, 6, "png"); + const result = await read("a.png", png, { images: processor }); + expect(result.output).toEqual([{ type: "image", data: toBase64(png), mimeType: "image/png" }]); + expect(onlyImage(await read("a.gif", gif(), { images: processor })).mimeType).toBe("image/gif"); + }); + + it("scales an image down to the maximum size, and says how to map coordinates back", async () => { + const result = await read("wide.png", encode(4000, 1000, "png"), { images: processor }); + const block = onlyImage(result); + expect(decodedImage(block.data)).toMatchObject({ width: 2000, height: 500 }); + expect(messages(result)).toEqual([ + `Read image file [${block.mimeType}]. Resized from 4000x1000 to 2000x500. Multiply coordinates by 2.00 to map them to the original.`, + ]); + }); + + it("re-encodes a PNG too large for the byte limit as JPEG at its size, and shrinks it when that is not enough", async () => { + const result = await read("big.png", noisyPng(1100, 1100), { images: processor }); + const jpeg = onlyImage(result); + expect(jpeg.mimeType).toBe("image/jpeg"); + expect(messages(result)).toEqual(["Read image file [image/jpeg]. Converted from image/png to image/jpeg."]); + expect(jpeg.data.length).toBeLessThanOrEqual(DEFAULT_IMAGE_LIMITS.maxBytes); + expect(decodedImage(jpeg.data)).toMatchObject({ width: 1100, height: 1100 }); + const tight = { ...DEFAULT_IMAGE_LIMITS, maxBytes: 200_000 }; + const shrunk = await processor.prepare(noisyPng(2000, 2000), "image/png", tight); + expect(shrunk!.data.length).toBeLessThanOrEqual(tight.maxBytes); + expect(shrunk!.resized?.from).toEqual({ width: 2000, height: 2000 }); + expect(shrunk!.resized!.to.width).toBeLessThan(2000); + }); + + it("keeps at least one pixel on each side of an extreme aspect ratio", async () => { + const result = await processor.prepare(encode(6000, 2, "png"), "image/png", DEFAULT_IMAGE_LIMITS); + expect(result?.resized).toEqual({ from: { width: 6000, height: 2 }, to: { width: 2000, height: 1 } }); + }); + + it("gives up when not even one pixel fits the byte limit", async () => { + const limits = { ...DEFAULT_IMAGE_LIMITS, maxBytes: 8 }; + expect(await processor.prepare(encode(4, 4, "png"), "image/png", limits)).toBeUndefined(); + }); + + it("keeps a JPEG that must be re-encoded a JPEG", async () => { + const result = await processor.prepare( + withOrientation(encode(40, 30, "jpeg"), 3), + "image/jpeg", + DEFAULT_IMAGE_LIMITS, + ); + expect(result?.mimeType).toBe("image/jpeg"); + }); + + it("converts a format providers do not take inline to PNG", async () => { + const result = await read("a.bmp", bmp(4, 4), { images: processor }); + expect(result.output).toMatchObject([{ type: "image", mimeType: "image/png" }]); + expect(messages(result)).toEqual(["Read image file [image/png]. Converted from image/bmp to image/png."]); + }); + + it("reports bytes it cannot decode as an error", async () => { + const broken = new Uint8Array([...encode(4, 4, "png").subarray(0, 40)]); + const result = await read("broken.png", broken, { images: processor }); + expect(result).toMatchObject({ output: [], isError: true }); + expect(messages(result)).toEqual(["broken.png is an image (image/png) that cannot be prepared for the model"]); + }); + + it("is what createCodingTools({ images }) gives read", async () => { + const tool = createCodingTools({ images: processor }).tools?.find((candidate) => candidate.name === "read"); + const result = await read("b.bmp", bmp(4, 4), { tool: tool as ReturnType }); + expect(result.output).toMatchObject([{ type: "image", mimeType: "image/png" }]); + }); +}); + +describe("EXIF orientation", () => { + const R: Rgb = [255, 0, 0]; + const G: Rgb = [0, 255, 0]; + const B: Rgb = [0, 0, 255]; + const W: Rgb = [255, 255, 255]; + const K: Rgb = [0, 0, 0]; + const Y: Rgb = [255, 255, 0]; + const stored = [ + [R, G, B], + [W, K, Y], + ]; + const mirror = (grid: readonly (readonly T[])[]): T[][] => grid.map((row) => [...row].reverse()); + // A quarter turn clockwise: the left column, read bottom to top, becomes the top row. + const turn = (grid: readonly (readonly T[])[]): T[][] => + grid[0]!.map((_, x) => grid.map((row) => row[x]!).reverse()); + // What each orientation shows, from its EXIF definition: mirror, then turn clockwise. + const upright: Record = { + 1: stored.map((row) => [...row]), + 2: mirror(stored), + 3: turn(turn(stored)), + 4: turn(turn(mirror(stored))), + 5: turn(turn(turn(mirror(stored)))), + 6: turn(stored), + 7: turn(mirror(stored)), + 8: turn(turn(turn(stored))), + }; + + for (let orientation = 1; orientation <= 8; orientation++) { + it(`turns orientation ${orientation} upright`, async () => { + const { rgba, width, height } = blocks(stored, 16); + const jpeg = withOrientation(encodeRgba(rgba, width, height, "jpeg"), orientation); + const prepared = await processor.prepare(jpeg, "image/jpeg", DEFAULT_IMAGE_LIMITS); + const image = decodedImage(prepared!.data); + const expected = upright[orientation]!; + expect({ width: image.width, height: image.height }).toEqual({ + width: expected[0]!.length * 16, + height: expected.length * 16, + }); + // The colour at the centre of each block, rounded to full channels against JPEG noise. + const seen = expected.map((row, by) => + row.map((_, bx) => { + const at = ((by * 16 + 8) * image.width + bx * 16 + 8) * 4; + return [...image.rgba.subarray(at, at + 3)].map((channel) => (channel > 127 ? 255 : 0)); + }), + ); + expect(seen).toEqual(expected); + }); + } + + it("reads big- and little-endian JPEG EXIF, after an APP0 segment, and 1 when absent or out of range", () => { + const jpeg = encode(2, 2, "jpeg"); + expect(exifOrientation(withOrientation(jpeg, 6))).toBe(6); + expect(exifOrientation(withOrientation(jpeg, 8, { little: true }))).toBe(8); + expect(exifOrientation(withOrientation(jpeg, 5, { app0: true }))).toBe(5); + expect(exifOrientation(withOrientation(jpeg, 9))).toBe(1); + expect(exifOrientation(jpeg)).toBe(1); + expect(exifOrientation(encode(2, 2, "png"))).toBe(1); + }); + + it("reads WebP EXIF chunks with and without the Exif prefix", () => { + expect(exifOrientation(webpWithOrientation(6, false))).toBe(6); + expect(exifOrientation(webpWithOrientation(3, true))).toBe(3); + }); +}); + +describe("imageDimensions", () => { + it("reads the size each format declares in its header", () => { + expect(imageDimensions(encode(7, 5, "png"), "image/png")).toEqual({ width: 7, height: 5 }); + expect(imageDimensions(encode(7, 5, "jpeg"), "image/jpeg")).toEqual({ width: 7, height: 5 }); + expect(imageDimensions(withOrientation(encode(7, 5, "jpeg"), 6, { app0: true }), "image/jpeg")).toEqual({ + width: 7, + height: 5, + }); + expect(imageDimensions(gif(300, 200), "image/gif")).toEqual({ width: 300, height: 200 }); + const webp = (type: string, data: number[]): Uint8Array => + new Uint8Array([ + ...Buffer.from("RIFF"), + 0, + 0, + 0, + 0, + ...Buffer.from("WEBP"), + ...Buffer.from(type), + 0, + 0, + 0, + 0, + ...data, + ...new Array(16).fill(0), + ]); + // VP8: frame tag, start code, then 14-bit width and height. + expect(imageDimensions(webp("VP8 ", [0, 0, 0, 0x9d, 0x01, 0x2a, 0x2c, 0x01, 0xc8, 0x00]), "image/webp")).toEqual({ + width: 300, + height: 200, + }); + // VP8L: signature, then width - 1 and height - 1 in 14 bits each. + const bits = 299 | (199 << 14); + expect( + imageDimensions( + webp("VP8L", [0x2f, bits & 0xff, (bits >> 8) & 0xff, (bits >> 16) & 0xff, bits >>> 24]), + "image/webp", + ), + ).toEqual({ width: 300, height: 200 }); + // VP8X: flags, then width - 1 and height - 1 in 24 bits each. + expect(imageDimensions(webp("VP8X", [0, 0, 0, 0, 0x2b, 0x01, 0, 0xc7, 0, 0]), "image/webp")).toEqual({ + width: 300, + height: 200, + }); + expect(imageDimensions(new Uint8Array([0xff, 0xd8, 0xff]), "image/jpeg")).toBeUndefined(); + }); +}); + +describe("toBase64", () => { + it("matches Buffer for every remainder length", () => { + for (const length of [0, 1, 2, 3, 4, 5, 10_000]) { + const bytes = Uint8Array.from({ length }, (_, index) => (index * 37) % 256); + expect(toBase64(bytes)).toBe(Buffer.from(bytes).toString("base64")); + } + }); +}); diff --git a/packages/durable/test/tools-read-differential.test.ts b/packages/durable/test/tools-read-differential.test.ts index ea846f7d1..f105a171e 100644 --- a/packages/durable/test/tools-read-differential.test.ts +++ b/packages/durable/test/tools-read-differential.test.ts @@ -18,15 +18,8 @@ function referenceRead(bytes: Uint8Array, path: string, offset: number | undefin const mimeType = detectSupportedImageMimeType(bytes); if (mimeType) { return { - output: [], - isError: true, - diagnostics: [ - { - severity: "error", - code: "unsupported_image", - message: `${path} is an image (${mimeType}); reading images is not supported`, - }, - ], + output: [{ type: "image", data: Buffer.from(bytes).toString("base64"), mimeType }], + diagnostics: [{ severity: "info", code: "image", message: `Read image file [${mimeType}].` }], }; } const textContent = new TextDecoder().decode(bytes); @@ -136,6 +129,7 @@ function apiFor(env: NodeExecutionEnv): ToolExecutionApi { outputWindow: undefined, diagnostic: () => {}, details: async () => {}, + agent: async () => ({}), } as unknown as ToolExecutionApi; } diff --git a/packages/durable/test/tools.test.ts b/packages/durable/test/tools.test.ts index d18d728d7..dba4d7c9d 100644 --- a/packages/durable/test/tools.test.ts +++ b/packages/durable/test/tools.test.ts @@ -73,6 +73,8 @@ function fakeApi(env: ExecutionEnv | undefined): { retainedOutput: () => ({ text: output.join(""), truncated: false }), diagnostic: (diagnostic: ToolDiagnostic) => diagnostics.push(diagnostic), details: async () => {}, + // No model: `read` treats it as one that sees images. + agent: async () => ({}), } as unknown as ToolExecutionApi; return { api, output, diagnostics }; } @@ -303,18 +305,15 @@ describe("durable tools", () => { expect(textOutput(result)).toBe("one\ntwo"); }); - it("reports images by content as unsupported", async () => { + it("returns images by content as one image block", async () => { const env = createEnv(); - const png = Uint8Array.from( - Buffer.from( - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGD4DwABBAEAX+XDSwAAAABJRU5ErkJggg==", - "base64", - ), - ); - getOrThrow(await env.writeFile("image.txt", png, BACKGROUND_CONTEXT)); + const data = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGD4DwABBAEAX+XDSwAAAABJRU5ErkJggg=="; + getOrThrow(await env.writeFile("image.txt", Uint8Array.from(Buffer.from(data, "base64")), BACKGROUND_CONTEXT)); const result = await run(createReadTool(), { path: "image.txt" }, env); - expect(result).toMatchObject({ output: [], isError: true }); - expect(diagnosticText(result)).toBe("image.txt is an image (image/png); reading images is not supported"); + expect(result.output).toEqual([{ type: "image", data, mimeType: "image/png" }]); + expect(result.isError).toBeUndefined(); + expect(diagnosticText(result)).toBe("Read image file [image/png]."); }); }); diff --git a/packages/durable/tsconfig.build.json b/packages/durable/tsconfig.build.json index 27422c97b..7befed510 100644 --- a/packages/durable/tsconfig.build.json +++ b/packages/durable/tsconfig.build.json @@ -9,5 +9,7 @@ "rootDir": "./src" }, "include": ["src/**/*.ts"], + // Ambient declarations the build needs; `exclude` below does not apply to `files`. + "files": ["src/images/photon-wasm.d.ts"], "exclude": ["node_modules", "dist", "**/*.d.ts", "src/**/*.d.ts"] } diff --git a/scripts/check-entry-graphs.mjs b/scripts/check-entry-graphs.mjs index 8cda4c37a..a619546df 100644 --- a/scripts/check-entry-graphs.mjs +++ b/scripts/check-entry-graphs.mjs @@ -46,8 +46,15 @@ const BUDGETS = { // Tool argument validation reaches TypeBox; provider-session creation reaches pi-ai's lean UUID utility; the // memory and SQLite storages share scan order and cursor handling. maxFiles: 63, - forbid: ["packages/ai/src/index.ts", "packages/ai/src/utils/typebox-helpers.ts"], + forbid: [ + "packages/ai/src/index.ts", + "packages/ai/src/utils/typebox-helpers.ts", + // Photon (WebAssembly) loads only through `./images`. + "packages/durable/src/images/", + ], }, + // The file tools reach their environment helpers and the Harness definitions they need, not the Harness itself. + "./tools": { maxFiles: 17, forbid: ["packages/durable/src/images/"] }, }, };