mirror of
https://github.com/badlogic/pi-mono.git
synced 2026-11-05 23:37:25 +00:00
feat(coding-agent): shrink codemode prompt and guide scripts back from errors
Move the codemode script reference, including the models API, into docs/codemode.md and keep only one line per global in the tool description. Declared tools get a one-line note on how scripts call them, and the codemode system prompt guidance and MCP server section are shorter. With the default tools a GPT-5.6 request drops from about 5,300 to 3,300 tokens. Errors now point the way back: unknown tools and models members name close matches, models.classify() and models.generateImages() validate their arguments, unknown or mistyped models point to getAvailableOfType(), store() overflow explains the store, and unshown generated images get a note.
This commit is contained in:
Armin Ronacher
parent
88ff80b986
commit
6f1072cc08
19 changed files with 607 additions and 222 deletions
|
|
@ -2,6 +2,14 @@
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- Added `renderToolOutputType()`, the type a tool call resolves to.
|
||||
|
||||
### Changed
|
||||
|
||||
- Reading a member of `tools` or of a global namespace that does not exist now throws an error naming the close matches, instead of returning `undefined`. Use `"name" in tools` to check for a tool. `store()` size errors explain what the store is for.
|
||||
|
||||
## [0.99.2] - 2026-09-30
|
||||
|
||||
### Changed
|
||||
|
|
|
|||
|
|
@ -143,7 +143,7 @@ export function renderToolSignature(
|
|||
tool.inputSchema === undefined
|
||||
? "unknown"
|
||||
: schemaToType(tool.inputSchema, { maxChars: options.inputMaxChars ?? DEFAULT_INPUT_SCHEMA_MAX_CHARS });
|
||||
return `${toCodemodeIdentifier(tool.name)}(args: ${input}): Promise<${outputType(tool.outputSchema)}>;`;
|
||||
return `${toCodemodeIdentifier(tool.name)}(args: ${input}): Promise<${renderToolOutputType(tool.outputSchema)}>;`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -174,7 +174,11 @@ export function mcpStructuredContentSchema(schema: CodemodeJsonSchema | undefine
|
|||
return isObject(structuredContent) || typeof structuredContent === "boolean" ? structuredContent : true;
|
||||
}
|
||||
|
||||
function outputType(schema: CodemodeJsonSchema | undefined): string {
|
||||
/**
|
||||
* The type a tool call resolves to: `CallToolResult<T>` for MCP output schemas (needs
|
||||
* {@link MCP_TYPESCRIPT_PREAMBLE}), the schema's type otherwise, and `unknown` without a schema.
|
||||
*/
|
||||
export function renderToolOutputType(schema: CodemodeJsonSchema | undefined): string {
|
||||
const structured = mcpStructuredContentSchema(schema);
|
||||
if (structured !== undefined) {
|
||||
const type = schemaToType(structured);
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ export {
|
|||
mcpStructuredContentSchema,
|
||||
type RenderDeclarationsOptions,
|
||||
renderDeclarations,
|
||||
renderToolOutputType,
|
||||
renderToolSample,
|
||||
renderToolSignature,
|
||||
schemaToType,
|
||||
|
|
|
|||
|
|
@ -113,6 +113,34 @@ export const PRELUDE_SOURCE: string = `(function (bridge, toolsJson, globalsJson
|
|||
Object.freeze(tools);
|
||||
Object.freeze(allTools);
|
||||
|
||||
// Reading a member that does not exist throws an error that names the close matches, instead of
|
||||
// a later "not a function". \`in\` checks still work.
|
||||
const comparable = (name) => name.toLowerCase().replace(/[^a-z0-9]/g, "");
|
||||
function guard(target, label, names, hint) {
|
||||
return new Proxy(target, {
|
||||
get(object, property, receiver) {
|
||||
if (typeof property !== "string" || property in object || property in Object.prototype || property === "then" || property === "toJSON") {
|
||||
return Reflect.get(object, property, receiver);
|
||||
}
|
||||
const wanted = comparable(property);
|
||||
const exact = names.filter((name) => comparable(name) === wanted);
|
||||
const close = exact.length > 0 ? exact : names.filter((name) => wanted && (comparable(name).includes(wanted) || wanted.includes(comparable(name))));
|
||||
let message = label + "." + property + " does not exist.";
|
||||
if (close.length > 0) message += " Did you mean " + close.slice(0, 5).map((name) => label + "." + name).join(", ") + "?";
|
||||
else if (names.length <= 20) message += " Available: " + names.join(", ") + ".";
|
||||
if (hint) message += " " + hint;
|
||||
message += ' Check for a member with "' + property + '" in ' + label + ".";
|
||||
throw new TypeErrorCtor(message);
|
||||
},
|
||||
});
|
||||
}
|
||||
const toolsProxy = guard(
|
||||
tools,
|
||||
"tools",
|
||||
allTools.map((tool) => tool.name),
|
||||
"ALL_TOOLS lists every tool; searchTools(query) finds tools by topic.",
|
||||
);
|
||||
|
||||
const namespaces = new Map();
|
||||
for (const { name, spread } of parse(globalsJson)) {
|
||||
const fn = caller("global", name, spread);
|
||||
|
|
@ -126,7 +154,9 @@ export const PRELUDE_SOURCE: string = `(function (bridge, toolsJson, globalsJson
|
|||
namespaces.get(namespace)[name.slice(dot + 1)] = fn;
|
||||
}
|
||||
for (const [namespace, members] of namespaces) {
|
||||
Object.defineProperty(globalThis, namespace, { value: Object.freeze(members), enumerable: true });
|
||||
Object.freeze(members);
|
||||
const value = guard(members, namespace, Object.keys(members));
|
||||
Object.defineProperty(globalThis, namespace, { value, enumerable: true });
|
||||
}
|
||||
|
||||
// key -> JSON text. Sizes count key and JSON characters.
|
||||
|
|
@ -135,6 +165,9 @@ export const PRELUDE_SOURCE: string = `(function (bridge, toolsJson, globalsJson
|
|||
let storedChars = 0;
|
||||
for (const [key, json] of stored) storedChars += key.length + json.length;
|
||||
|
||||
const STORE_HINT =
|
||||
"store() is for small state such as IDs or summaries. Show images with image(), keep large data in variables, or write it to a file with a tool.";
|
||||
|
||||
function checkKey(name, key) {
|
||||
if (typeof key !== "string") throw new TypeError(name + "() key must be a string");
|
||||
}
|
||||
|
|
@ -158,11 +191,17 @@ export const PRELUDE_SOURCE: string = `(function (bridge, toolsJson, globalsJson
|
|||
throw new TypeError("store(" + stringify(key) + ") value is not JSON-serializable");
|
||||
}
|
||||
if (json.length > ${MAX_STORE_VALUE_CHARS}) {
|
||||
throw new RangeError("store(" + stringify(key) + ") value exceeds ${MAX_STORE_VALUE_CHARS} characters of JSON");
|
||||
throw new RangeError(
|
||||
"store(" + stringify(key) + ") value has " + json.length + " characters of JSON, more than the limit of ${MAX_STORE_VALUE_CHARS}. " +
|
||||
STORE_HINT,
|
||||
);
|
||||
}
|
||||
const next = storedChars - previous + key.length + json.length;
|
||||
if (next > ${MAX_STORE_TOTAL_CHARS}) {
|
||||
throw new RangeError("store is full: stored values would exceed ${MAX_STORE_TOTAL_CHARS} characters of JSON");
|
||||
throw new RangeError(
|
||||
"store is full: stored values would exceed ${MAX_STORE_TOTAL_CHARS} characters of JSON. Delete keys with store(key, undefined). " +
|
||||
STORE_HINT,
|
||||
);
|
||||
}
|
||||
stored.set(key, json);
|
||||
storedChars = next;
|
||||
|
|
@ -278,7 +317,7 @@ export const PRELUDE_SOURCE: string = `(function (bridge, toolsJson, globalsJson
|
|||
}
|
||||
Object.freeze(console);
|
||||
|
||||
Object.defineProperty(globalThis, "tools", { value: tools, enumerable: true });
|
||||
Object.defineProperty(globalThis, "tools", { value: toolsProxy, enumerable: true });
|
||||
Object.defineProperty(globalThis, "ALL_TOOLS", { value: allTools, enumerable: true });
|
||||
Object.defineProperty(globalThis, "console", { value: console, enumerable: true });
|
||||
Object.defineProperty(globalThis, "text", { value: text, enumerable: true });
|
||||
|
|
@ -306,7 +345,7 @@ export const PRELUDE_SOURCE: string = `(function (bridge, toolsJson, globalsJson
|
|||
run(fn) {
|
||||
let promise;
|
||||
try {
|
||||
promise = fn(tools, console);
|
||||
promise = fn(toolsProxy, console);
|
||||
} catch (error) {
|
||||
done(false, describeError(error));
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -332,6 +332,22 @@ describe("tools", () => {
|
|||
expect(aborted).toBe(true);
|
||||
});
|
||||
|
||||
it("names close matches when a script reads a tool that does not exist", async () => {
|
||||
const sandbox = createSandbox([echo, { name: "web-search", execute: () => "" }]);
|
||||
const attempt = async (expression: string) => {
|
||||
const result = await sandbox.execute(`return ${expression};`);
|
||||
return result.ok ? result.value : result.error.message;
|
||||
};
|
||||
expect(await attempt("tools.Echo")).toBe(
|
||||
'tools.Echo does not exist. Did you mean tools.echo? ALL_TOOLS lists every tool; searchTools(query) finds tools by topic. Check for a member with "Echo" in tools.',
|
||||
);
|
||||
expect(await attempt("tools.websearch")).toContain("Did you mean tools.web_search?");
|
||||
expect(await attempt("tools.nothing")).toContain("Available: echo, web_search.");
|
||||
expect(
|
||||
await attempt("['echo' in tools, 'nothing' in tools, String(tools.toString), JSON.stringify(tools)]"),
|
||||
).toEqual([true, false, "undefined", "{}"]);
|
||||
});
|
||||
|
||||
it("supports register and unregister between executions", async () => {
|
||||
const sandbox = createSandbox();
|
||||
sandbox.registerTool(echo);
|
||||
|
|
@ -339,7 +355,7 @@ describe("tools", () => {
|
|||
expect(sandbox.tools.map((tool) => tool.name)).toEqual(["echo"]);
|
||||
expect(await sandbox.execute("return await tools.echo('a')")).toMatchObject({ ok: true, value: "a" });
|
||||
expect(sandbox.unregisterTool("echo")).toBe(true);
|
||||
expect(await sandbox.execute("return typeof tools.echo")).toMatchObject({ ok: true, value: "undefined" });
|
||||
expect(await sandbox.execute("return 'echo' in tools")).toMatchObject({ ok: true, value: false });
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -392,6 +408,15 @@ describe("store and load", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("explains oversized writes", async () => {
|
||||
const sandbox = createSandbox();
|
||||
const result = await sandbox.execute(`store("img", "x".repeat(300 * 1024));`);
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) return;
|
||||
expect(result.error.message).toContain('store("img") value has 307202 characters of JSON');
|
||||
expect(result.error.message).toContain("Show images with image()");
|
||||
});
|
||||
|
||||
it("reserves the store and load names", () => {
|
||||
const execute = () => undefined;
|
||||
expect(() => new CodemodeSandbox({ globals: [{ name: "store", execute }] })).toThrow(/Invalid global/);
|
||||
|
|
@ -431,13 +456,30 @@ describe("globals", () => {
|
|||
await models.list("classifier", undefined, 3);
|
||||
await models.list();
|
||||
try { models.extra = 1; } catch {}
|
||||
return [Object.keys(models), await models.first("a", "ignored"), typeof models.extra];
|
||||
return [Object.keys(models), await models.first("a", "ignored"), "extra" in models];
|
||||
`);
|
||||
expect(result).toMatchObject({ ok: true, value: [["list", "first"], "a", "undefined"] });
|
||||
expect(result).toMatchObject({ ok: true, value: [["list", "first"], "a", false] });
|
||||
// undefined array elements become null in the JSON round trip.
|
||||
expect(seen).toEqual([["classifier", null, 3], []]);
|
||||
});
|
||||
|
||||
it("names the members of a namespace when a script reads one that does not exist", async () => {
|
||||
const execute = () => undefined;
|
||||
const sandbox = new CodemodeSandbox({
|
||||
globals: [
|
||||
{ name: "models.classify", execute },
|
||||
{ name: "models.generateImages", execute },
|
||||
],
|
||||
});
|
||||
sandboxes.push(sandbox);
|
||||
const result = await sandbox.execute("await models.generateImage();");
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) return;
|
||||
expect(result.error.message).toBe(
|
||||
'models.generateImage does not exist. Did you mean models.generateImages? Check for a member with "generateImage" in models.',
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects invalid and reserved global names", () => {
|
||||
const execute = () => undefined;
|
||||
for (const name of ["a.b.c", "a.", ".a", "tools.x", "store.x", "a.not-valid"]) {
|
||||
|
|
@ -640,8 +682,8 @@ describe("escape hatches", () => {
|
|||
try { tools.echo = () => 'nope'; } catch {}
|
||||
try { tools.extra = () => 'nope'; } catch {}
|
||||
try { globalThis.tools = null; } catch {}
|
||||
return [typeof tools.extra, await tools.echo('still')];
|
||||
return ["extra" in tools, await tools.echo('still')];
|
||||
`);
|
||||
expect(result).toMatchObject({ ok: true, value: ["undefined", "still"] });
|
||||
expect(result).toMatchObject({ ok: true, value: [false, "still"] });
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@
|
|||
- `/login` now offers "Sign in with Radius" at the top level, as the last option, with its status. After a Radius sign-in, `/login` offers to configure the Radius MCP server in the global `mcp.json` with `"auth": { "provider": "radius" }` and reloads. Cancelling a login returns to the menu it was started from.
|
||||
- The provider docs page is renamed to [Providers](docs/providers.md), its "Cloud Providers" section is now "Provider Specific Config", and it documents Radius first.
|
||||
- MCP OAuth credentials are now stored per server name and URL, so MCP servers with the same URL can sign in with different accounts. Credentials stored by URL alone move to the first server that uses them ([#10252](https://github.com/earendil-works/pi/issues/10252)).
|
||||
- Codemode costs far fewer prompt tokens: with the default tools and codemode active, a GPT-5.6 request shrinks from about 5,300 to 3,300 tokens. The `codemode` description lists the script globals in one line each and points to the new [Codemode](docs/codemode.md) reference for the `models` API, which the model reads when it needs it. Declared tools say in one line how scripts call them and what the call resolves to, instead of repeating their full declaration, and the system prompt's codemode guidance and MCP server section are shorter.
|
||||
- Codemode errors now say how to recover: reading a tool or `models` member that does not exist names the close matches (`tools.Bash` suggests `tools.bash`), `models.classify()` and `models.generateImages()` reject malformed arguments with the expected shape, an unknown model points to `models.getAvailableOfType()`, an oversized `store()` value explains what the store is for, and a script that generates images without showing them gets a note. Scripts that probed for a tool with `typeof tools.name` must use `"name" in tools`.
|
||||
|
||||
### Fixed
|
||||
|
||||
|
|
|
|||
|
|
@ -165,17 +165,7 @@ Codemode is useful without MCP: scripts can run several tool calls in parallel,
|
|||
|
||||
### How codemode works
|
||||
|
||||
Codemode scripts run in a QuickJS sandbox that can only reach the other tools, through `tools.<name>(args)`; `ALL_TOOLS` lists them. Output comes from `text(value)`, `image(dataUrlOrImageContent)`, `console.*`, and a top-level `return value`; `exit()` ends the script early. The result starts with `Script completed` or `Script failed`, the wall time, and the output; a failed script keeps its partial output, followed by `Script error:` and the error.
|
||||
|
||||
A script may start with an options line such as `// @options: {"max_output_tokens": 2000, "timeout_ms": 60000}`. `max_output_tokens` (default 10000) limits the output: longer output keeps its start and end, and the full text is written to a temp file whose path is included in the result. `timeout_ms` is a hard deadline, unset by default.
|
||||
|
||||
While `codemode` is active, `codemode.mode` in [settings](settings.md#tools) decides how the other tools are presented. With `on` (default) declared tools keep being declared and their descriptions show how to call them from scripts. With `only` they are hidden from the model and listed in the `codemode` description instead, so the model calls them through scripts.
|
||||
|
||||
The `codemode` description lists the callable tools with their TypeScript declarations, grouped by namespace (for example one MCP server). Tools with `deferred` exposure, which includes MCP tools with the default `codemode` exposure, are not listed and do not affect the description, so it stays the same while MCP servers connect. Declarations share a budget of 3000 estimated tokens (`codemode.inlineBudget` in [settings](settings.md#tools)). Scripts find the rest with `await searchTools(query, { limit, namespace })`, which ranks tools with BM25, and `await describeTool(name)`, or by filtering `ALL_TOOLS`. `await describeNamespace(name)` returns a namespace's description, its instructions (for MCP servers, the server instructions), and the names of its tools.
|
||||
|
||||
Tools with an output schema resolve to structured values: `bash` to `{ output, truncated, full_output_path?, exit_code, wall_time_seconds }`, also for non-zero exit codes, and MCP tools to their `CallToolResult`. Other tools resolve to their text output. The `output` of `bash` is not limited to the 2000 lines or 50KB the model sees: it holds up to 1 MiB, and longer output keeps its first and last 512 KiB around an omission marker, with `truncated` set and the full output in `full_output_path`.
|
||||
|
||||
`store(key, value)` and `load(key)` keep JSON values across `codemode` calls: each successful script that stores values appends a `codemode-store` custom entry to the session, so resumed sessions keep the values and each branch sees only the values written on its path. Scripts can also use `models`: `getModelsOfType`, `getAvailableOfType`, and `getModelOfType` list the model catalog, `classify(model, context)` runs a classifier model, and `generateImages(model, context)` runs an image model. Both use the session's credentials, with at most four such calls in flight per script.
|
||||
Scripts run in a QuickJS sandbox and reach the other tools through `tools.<name>(args)`. [Codemode](codemode.md) describes the script API, how tools are listed and found, the `store()` and `models` globals, and the limits.
|
||||
|
||||
### Tool search
|
||||
|
||||
|
|
|
|||
207
packages/coding-agent/docs/codemode.md
Normal file
207
packages/coding-agent/docs/codemode.md
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
# Codemode
|
||||
|
||||
The `codemode` tool lets the model write a JavaScript script that calls pi's other tools and runs non-LLM models, such as classifiers and image models. Only the script's output reaches the model, so a script can run calls in parallel and filter large results before the model sees them. To turn it on, see [Enable codemode](cli.md#enable-codemode).
|
||||
|
||||
## Scripts
|
||||
|
||||
The tool input is raw JavaScript source, not JSON and not a markdown code fence. It runs as the body of an async function in a QuickJS sandbox, so top-level `await` and `return` work. The sandbox has no Node APIs, file system, network, or timers; scripts reach the outside world only through tools and `models`.
|
||||
|
||||
A script may start with an options line:
|
||||
|
||||
```js
|
||||
// @options: {"max_output_tokens": 2000, "timeout_ms": 60000}
|
||||
```
|
||||
|
||||
- `max_output_tokens` (default 10000) limits the output. Longer output keeps its start and end, and the full text is written to a temp file whose path is included in the result.
|
||||
- `timeout_ms` is a hard deadline for the whole script. It is unset by default. Image generation can take minutes, so do not set a short deadline for scripts that generate images.
|
||||
|
||||
The result starts with `Script completed` or `Script failed`, the wall time, and the output. A failed script keeps its partial output, followed by `Script error:` and the error. Tool calls are real: calls made before a failure are not undone. Calls still running when the script ends are cancelled, and unawaited promises are discarded.
|
||||
|
||||
## Globals
|
||||
|
||||
| Global | Purpose |
|
||||
|---|---|
|
||||
| `tools.<name>(args)` | Call a tool. See [Call tools](#call-tools). |
|
||||
| `text(value)` | Add a text item to the output. Strings are added as is, other values as JSON. |
|
||||
| `image(value)` | Add an image to the output: a base64 `data:` URL, an `{ image_url }` object, or an image block `{ type: "image", data, mimeType }` such as those returned by MCP tools and `models.generateImages()`. Remote URLs are not supported. PNG, JPEG, GIF, and WebP are accepted. |
|
||||
| `console.log(...)` | Like `text()`; `info`, `warn`, `error`, and `debug` do the same. |
|
||||
| `return value` | A top-level `return` adds the value like `text()`. |
|
||||
| `exit()` | End the script successfully. |
|
||||
| `store(key, value)` / `load(key)` | Keep small JSON values across `codemode` calls. See [Store values](#store-values). |
|
||||
| `ALL_TOOLS` | Every callable tool as `{ name, description }`, including tools the description does not list. |
|
||||
| `searchTools(query, { limit?, namespace? })` | Rank callable tools by relevance (BM25, default limit 8). Resolves to `{ name, description }[]`. |
|
||||
| `describeTool(name)` | Resolves to a tool's description and TypeScript declaration, or `undefined`. |
|
||||
| `describeNamespace(name)` | Resolves to `{ name, description?, instructions?, tools }` for a namespace such as an MCP server, or `undefined`. |
|
||||
| `models` | List and run non-LLM models. See [Models](#models). |
|
||||
|
||||
## Call tools
|
||||
|
||||
Every tool the session can call is a method of `tools`, named by its identifier: characters that are not valid in a JavaScript identifier become `_`, so the MCP tool `mcp__dev-radius__search` is `tools.mcp__dev_radius__search`. Each method takes one object with the tool's arguments.
|
||||
|
||||
What a call resolves to depends on the tool:
|
||||
|
||||
- Tools with an output schema resolve to a structured value. `bash` resolves to `{ output, truncated, full_output_path?, exit_code, wall_time_seconds }`, also for non-zero exit codes. Its `output` is not limited to the 2000 lines or 50KB the model sees: it holds up to 1 MiB, and longer output keeps its first and last 512 KiB around an omission marker, with `truncated` set and the full output in `full_output_path`.
|
||||
- MCP tools resolve to their `CallToolResult`, including `isError` and `structuredContent`.
|
||||
- Other tools, such as `read`, `edit`, and `write`, resolve to their text output.
|
||||
|
||||
A call that fails, is blocked, or gets invalid arguments rejects with an `Error` that carries the tool's error text. Use `Promise.allSettled()` to keep the results of the calls that succeed.
|
||||
|
||||
The `codemode` description lists tools with their TypeScript declarations, grouped by namespace (for example one MCP server). Tools with `deferred` exposure, which includes MCP tools with the default `codemode` exposure, are not listed, so the description stays the same while MCP servers connect. Listed declarations share a budget of 3000 estimated tokens (`codemode.inlineBudget` in [settings](settings.md#tools)). Scripts find the other tools with `searchTools()`, `describeTool()`, `describeNamespace()`, or by filtering `ALL_TOOLS`.
|
||||
|
||||
While `codemode` is active, `codemode.mode` in [settings](settings.md#tools) decides how the other tools are presented. With `on` (default) declared tools stay declared, and their descriptions say how to call them from scripts. With `only` they are hidden from the model and listed in the `codemode` description instead, so the model calls them through scripts.
|
||||
|
||||
## Store values
|
||||
|
||||
`store(key, value)` keeps a JSON value under a string key for later `codemode` calls; storing `undefined` deletes the key. `load(key)` returns the value, or `undefined`. Writes are kept only when the script succeeds: each successful script that stores values appends a `codemode-store` custom entry to the session, so resumed sessions keep the values and each branch sees only the values written on its path.
|
||||
|
||||
The store is for small state such as IDs, cursors, or summaries. One value may have at most 262144 characters of JSON and all values together at most 1048576. Do not store image data; show images with `image()` or write them to a file with a tool.
|
||||
|
||||
## Models
|
||||
|
||||
`models` reaches the model catalog and runs non-LLM models with the session's credentials: classifiers, which answer typed questions about JSON state, and image models, which generate images. Chat models are listed but cannot be run from scripts. Which classifier and image models exist is described in [Use classifier models](models.md#use-classifier-models) and [Use image models](models.md#use-image-models).
|
||||
|
||||
```ts
|
||||
type ModelType = "chat" | "image" | "classifier";
|
||||
|
||||
/** A catalog entry. `provider` and `id` identify it; other fields depend on the type. */
|
||||
interface ModelInfo {
|
||||
type?: ModelType;
|
||||
provider: string;
|
||||
id: string;
|
||||
name: string;
|
||||
api: string;
|
||||
input: ("text" | "image")[];
|
||||
contextWindow?: number;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
declare const models: {
|
||||
/** Every known model of a type, optionally for one provider. */
|
||||
getModelsOfType(type: ModelType, provider?: string): Promise<ModelInfo[]>;
|
||||
/** Models of a type whose provider has working credentials. */
|
||||
getAvailableOfType(type: ModelType, provider?: string): Promise<ModelInfo[]>;
|
||||
/** One catalog entry, or undefined. */
|
||||
getModelOfType(type: ModelType, provider: string, id: string): Promise<ModelInfo | undefined>;
|
||||
/** Answer `context.questions` about `context.state`; answers are in `result.answers` by question ID. */
|
||||
classify(model: ModelInfo, context: ClassifierContext): Promise<ClassifierResult>;
|
||||
/** Generate images from `context.input` text and image blocks; show `result.output` blocks with image(). Can take minutes. */
|
||||
generateImages(model: ModelInfo, context: ImagesContext): Promise<ImagesResult>;
|
||||
};
|
||||
```
|
||||
|
||||
`classify()` and `generateImages()` use only the `provider` and `id` of `model`, so `{ provider, id }` works as well. They do not throw on provider errors: check `stopReason` and `errorMessage`. At most four such calls run at once per script; more calls wait for a free slot, so `Promise.all()` over many items is fine. Their usage is added to the `codemode` tool result and counts toward the session cost.
|
||||
|
||||
Model IDs differ between providers, for example `typesafe/jev-latest` and `openrouter/typesafe/jev-1.13`. Use `models.getAvailableOfType(type)` to find the IDs that work with the current credentials.
|
||||
|
||||
### Classify
|
||||
|
||||
```ts
|
||||
interface ClassifierContext {
|
||||
/** The data to classify. */
|
||||
state: Record<string, unknown>;
|
||||
/** Questions by ID. One call answers all of them. */
|
||||
questions: Record<string, ClassifierQuestion>;
|
||||
}
|
||||
|
||||
type ClassifierQuestion =
|
||||
/** Pick one label. `criteria` maps each label to what it means. */
|
||||
| { type: "choice"; instructions: string; criteria: Record<string, string> }
|
||||
/** Score on an ordered scale. `criteria` describes each level, lowest first. */
|
||||
| { type: "score"; instructions: string; criteria: string[] }
|
||||
/** Yes or no. */
|
||||
| { type: "bool"; instructions: string; criteria: { true: string; false: string } };
|
||||
|
||||
interface ClassifierResult {
|
||||
provider: string;
|
||||
model: string;
|
||||
/** Answers by question ID. */
|
||||
answers: Record<string, ClassifierAnswer>;
|
||||
usage?: ModelUsage;
|
||||
stopReason: "stop" | "error" | "aborted";
|
||||
errorMessage?: string;
|
||||
}
|
||||
|
||||
type ClassifierAnswer =
|
||||
| { type: "choice"; choice: string; probabilities: Record<string, number>; confidence: number }
|
||||
/** `score` is the expected level index, from 0 to `criteria.length - 1`. */
|
||||
| { type: "score"; score: number; confidence: number }
|
||||
/** Probability of `true`. */
|
||||
| { type: "bool"; probability: number };
|
||||
|
||||
/** Token counts and cost in USD, when the service reports them. */
|
||||
type ModelUsage = { input: number; output: number; totalTokens: number; cost: { total: number } };
|
||||
```
|
||||
|
||||
Classify several items by calling `classify()` once per item. This script sorts feedback messages, for example ones a tool returned earlier in the script:
|
||||
|
||||
```js
|
||||
const jev = await models.getModelOfType("classifier", "typesafe", "jev-latest");
|
||||
const results = await Promise.all(
|
||||
messages.map((message) =>
|
||||
models.classify(jev, {
|
||||
state: { message },
|
||||
questions: {
|
||||
sentiment: {
|
||||
type: "choice",
|
||||
instructions: "How does the user feel about the product?",
|
||||
criteria: { positive: "Satisfied or happy", negative: "Unhappy or frustrated", neutral: "Neither" },
|
||||
},
|
||||
urgency: {
|
||||
type: "score",
|
||||
instructions: "How urgently does this need a reply?",
|
||||
criteria: ["no reply needed", "reply this week", "reply today"],
|
||||
},
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
return results.map((result, i) =>
|
||||
result.stopReason === "stop"
|
||||
? { message: messages[i], sentiment: result.answers.sentiment.choice, urgency: result.answers.urgency.score }
|
||||
: { message: messages[i], error: result.errorMessage },
|
||||
);
|
||||
```
|
||||
|
||||
### Generate images
|
||||
|
||||
```ts
|
||||
interface ImagesContext {
|
||||
/** The prompt as text blocks, plus image blocks to edit or use as references. */
|
||||
input: (TextBlock | ImageBlock)[];
|
||||
}
|
||||
|
||||
interface ImagesResult {
|
||||
provider: string;
|
||||
model: string;
|
||||
/** Generated images, and text blocks for models that also return text. */
|
||||
output: (TextBlock | ImageBlock)[];
|
||||
usage?: ModelUsage;
|
||||
stopReason: "stop" | "error" | "aborted";
|
||||
errorMessage?: string;
|
||||
}
|
||||
|
||||
type TextBlock = { type: "text"; text: string };
|
||||
/** `data` is base64. */
|
||||
type ImageBlock = { type: "image"; data: string; mimeType: string };
|
||||
```
|
||||
|
||||
Show generated images with `image(block)`. Do not print `data` with `text()`, `console`, or `return`: it is large and the model cannot read it as text. Generated images are not saved to disk; to keep one, write it to a file with a tool.
|
||||
|
||||
```js
|
||||
// @options: {"timeout_ms": 300000}
|
||||
const painter = await models.getModelOfType("image", "openrouter", "google/gemini-2.5-flash-image");
|
||||
const result = await models.generateImages(painter, {
|
||||
input: [{ type: "text", text: "A red fox in the snow, watercolor" }],
|
||||
});
|
||||
if (result.stopReason !== "stop") return result.errorMessage;
|
||||
for (const block of result.output) {
|
||||
if (block.type === "image") image(block);
|
||||
else text(block.text);
|
||||
}
|
||||
```
|
||||
|
||||
## Limits
|
||||
|
||||
- A script's VM has 256 MB of memory. Running out throws `InternalError: out of memory`; filter or aggregate large data instead of accumulating it.
|
||||
- A script that waits on a promise that can never settle (no tool call pending) fails immediately, since there are no timers.
|
||||
- Scripts cannot start other `codemode` scripts.
|
||||
|
|
@ -136,6 +136,10 @@
|
|||
"title": "CLI",
|
||||
"path": "cli.md"
|
||||
},
|
||||
{
|
||||
"title": "Codemode",
|
||||
"path": "codemode.md"
|
||||
},
|
||||
{
|
||||
"title": "Slash Commands",
|
||||
"path": "slash-commands.md"
|
||||
|
|
|
|||
|
|
@ -131,6 +131,8 @@ const result = await models.classify(jev, {
|
|||
return result.answers;
|
||||
```
|
||||
|
||||
[Codemode](codemode.md#classify) describes the question and answer types.
|
||||
|
||||
When the service reports token counts, as all System One services do, `result.usage` carries them with their cost. Pi adds the usage of a script's classifier calls to the `codemode` tool result, so it counts toward the session cost in the footer and `/session`. The cost uses the model's catalog price; models without one, such as TypeSafe's direct `jev-latest`, report tokens at no cost.
|
||||
|
||||
Extensions call classifiers through `ctx.modelRegistry.classify()`, without codemode. [Virtual models](virtual-models.md#route-requests) can use them to route requests; see the `jev-router.ts` example.
|
||||
|
|
@ -150,7 +152,7 @@ if (result.stopReason !== "stop") return result.errorMessage;
|
|||
for (const block of result.output) if (block.type === "image") image(block);
|
||||
```
|
||||
|
||||
`input` can also contain `{ type: "image", data, mimeType }` blocks to edit or use as references. Pi adds the usage of a script's image calls to the `codemode` tool result, like classifier calls. Generated images are not saved to disk.
|
||||
`input` can also contain `{ type: "image", data, mimeType }` blocks to edit or use as references. Pi adds the usage of a script's image calls to the `codemode` tool result, like classifier calls. Generated images are not saved to disk. [Codemode](codemode.md#generate-images) describes the full API.
|
||||
|
||||
Extensions generate images through `ctx.modelRegistry.generateImages()`, without codemode.
|
||||
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ See [Choose a Model](models.md) for model selection and thinking controls.
|
|||
| Setting | Type | Default | Description |
|
||||
|---|---|---|---|
|
||||
| `defaultTools` | `string[]` | `read`, `bash`, `edit`, `write` | Tools enabled at startup. Plain names replace the defaults; `+name` adds a tool and `-name` removes one. An empty array disables all built-in tools but not extension or SDK tools. |
|
||||
| `codemode.mode` | `"on"` \| `"only"` | `"on"` | How the `codemode` tool presents tools while it is active. `on`: declared tools get their `codemode` declaration appended to their description, and `codemode` lists only tools that are not declared. `only`: `codemode` lists every tool scripts can call, and active built-in and extension tools are hidden from the model, so it reaches them through `codemode`. |
|
||||
| `codemode.mode` | `"on"` \| `"only"` | `"on"` | How the `codemode` tool presents tools while it is active. `on`: declared tools get a note on calling them from scripts appended to their description, and `codemode` lists only tools that are not declared. `only`: `codemode` lists every tool scripts can call, and active built-in and extension tools are hidden from the model, so it reaches them through `codemode`. |
|
||||
| `codemode.inlineBudget` | number | `3000` | Estimated tokens (characters / 4) the `codemode` tool's description may spend on tool declarations. Tools that do not fit are left out and found with `searchTools()`. `0` lists only namespaces. |
|
||||
|
||||
Available built-in tools are `read`, `bash`, `powershell`, `edit`, `write`, `grep`, `find`, and `ls`. `defaultTools` can also name `codemode` and `tool_search`, which built-in extensions register inactive, and other extension tools registered inactive.
|
||||
|
|
|
|||
|
|
@ -93,8 +93,8 @@ export interface WarningSettings {
|
|||
|
||||
/**
|
||||
* How the codemode tool presents tools while it is active.
|
||||
* - `on`: declared tools that scripts can call get their codemode declaration appended to their
|
||||
* description; the codemode description lists only the tools without `direct` exposure.
|
||||
* - `on`: declared tools that scripts can call get a note on calling them from scripts appended to
|
||||
* their description; the codemode description lists only the tools without `direct` exposure.
|
||||
* - `only`: the codemode description lists every tool scripts can call, and active `direct` tools are
|
||||
* not declared to the model.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -155,7 +155,7 @@ export function buildSystemPromptSections(input: BuildSystemPromptOptions): Syst
|
|||
- Additional docs: ${getDocsPath()}
|
||||
- Examples: ${getExamplesPath()} (extensions, custom tools, SDK)
|
||||
- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory
|
||||
- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md)
|
||||
- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md)
|
||||
- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing
|
||||
- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)`;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -54,12 +54,9 @@ export type BashToolInput = Static<typeof bashSchema>;
|
|||
* `output` is not limited like the model-facing output: callers decide how much of it reaches the model.
|
||||
*/
|
||||
const bashOutputSchema = Type.Object({
|
||||
output: Type.String({
|
||||
description:
|
||||
"Combined stdout and stderr, up to 1 MiB. Longer output keeps its first and last 512 KiB around an omission marker.",
|
||||
}),
|
||||
truncated: Type.Boolean({ description: "Whether `output` omits part of the command output" }),
|
||||
full_output_path: Type.Optional(Type.String({ description: "Temp file with the full output, when truncated" })),
|
||||
output: Type.String({ description: "Combined stdout and stderr, possibly truncated" }),
|
||||
truncated: Type.Boolean(),
|
||||
full_output_path: Type.Optional(Type.String({ description: "Full output, when truncated" })),
|
||||
exit_code: Type.Number(),
|
||||
wall_time_seconds: Type.Number(),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ import type { SessionEntry } from "../../core/session-manager.ts";
|
|||
import { combineUsage } from "../../core/usage-totals.ts";
|
||||
import { Bm25Ranker, createToolSearchDocument, DEFAULT_TOOL_SEARCH_LIMIT } from "../tool-search/tool.ts";
|
||||
import {
|
||||
CODEMODE_DOCS_PATH,
|
||||
CODEMODE_STORE_ENTRY_TYPE,
|
||||
type CodemodeModelRuntime,
|
||||
type CodemodeNestedCall,
|
||||
|
|
@ -41,7 +42,6 @@ import {
|
|||
type CodemodeToolInput,
|
||||
type CodemodeToolOptions,
|
||||
getCodemodeCallableTools,
|
||||
MODEL_GLOBAL_DECLARATIONS,
|
||||
toCodemodeDeclaration,
|
||||
} from "./tool.ts";
|
||||
|
||||
|
|
@ -95,6 +95,93 @@ function toModelInfo(model: AnyModel): Record<string, unknown> {
|
|||
return info;
|
||||
}
|
||||
|
||||
/** `an image`, `a classifier`. */
|
||||
function withArticle(word: string): string {
|
||||
return `${/^[aeiou]/.test(word) ? "an" : "a"} ${word}`;
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
/** A script value in an error message: `undefined`, `a string`, `an array`, or its keys (`{ prompt }`). */
|
||||
function describeValue(value: unknown): string {
|
||||
if (value === undefined || value === null) return String(value);
|
||||
if (Array.isArray(value)) return value.length === 0 ? "an empty array" : "an array";
|
||||
if (typeof value === "object") {
|
||||
const keys = Object.keys(value);
|
||||
if (keys.length === 0) return "{}";
|
||||
return `{ ${keys.slice(0, 6).join(", ")}${keys.length > 6 ? ", ..." : ""} }`;
|
||||
}
|
||||
return typeof value === "string" ? "a string" : `a ${typeof value}`;
|
||||
}
|
||||
|
||||
const CLASSIFIER_CONTEXT_SHAPE =
|
||||
'{ state: { ... }, questions: { <id>: { type: "choice", instructions, criteria: { <label>: <meaning> } } | { type: "score", instructions, criteria: [<lowest level>, ..., <highest level>] } | { type: "bool", instructions, criteria: { true: <meaning>, false: <meaning> } } } }';
|
||||
|
||||
/** Check a script's classifier context, so mistakes fail with the expected shape instead of a provider error. */
|
||||
function checkClassifierContext(context: unknown): ClassifierContext {
|
||||
const fail = (problem: string) =>
|
||||
new Error(
|
||||
`models.classify() ${problem}. Expected context: ${CLASSIFIER_CONTEXT_SHAPE}. See "Classify" in ${CODEMODE_DOCS_PATH}.`,
|
||||
);
|
||||
if (!isRecord(context)) throw fail(`expects a context object as its second argument, got ${describeValue(context)}`);
|
||||
if (!isRecord(context.state)) throw fail(`context.state must be an object, got ${describeValue(context.state)}`);
|
||||
const { questions } = context;
|
||||
if (!isRecord(questions) || Object.keys(questions).length === 0) {
|
||||
throw fail(`context.questions must map question IDs to questions, got ${describeValue(questions)}`);
|
||||
}
|
||||
const isStrings = (values: unknown[]) => values.length > 0 && values.every((value) => typeof value === "string");
|
||||
for (const [id, question] of Object.entries(questions)) {
|
||||
const at = `context.questions.${id}`;
|
||||
if (!isRecord(question)) throw fail(`${at} must be a question object, got ${describeValue(question)}`);
|
||||
if (typeof question.instructions !== "string") throw fail(`${at}.instructions must be a string`);
|
||||
const { criteria } = question;
|
||||
if (question.type === "choice") {
|
||||
if (!isRecord(criteria) || !isStrings(Object.values(criteria))) {
|
||||
throw fail(`${at} is a "choice" question, so criteria must map each label to its meaning`);
|
||||
}
|
||||
} else if (question.type === "score") {
|
||||
if (!Array.isArray(criteria) || !isStrings(criteria)) {
|
||||
throw fail(`${at} is a "score" question, so criteria must list the levels as strings, lowest first`);
|
||||
}
|
||||
} else if (question.type === "bool") {
|
||||
if (!isRecord(criteria) || typeof criteria.true !== "string" || typeof criteria.false !== "string") {
|
||||
throw fail(`${at} is a "bool" question, so criteria must be { true: string, false: string }`);
|
||||
}
|
||||
} else {
|
||||
throw fail(`${at}.type must be "choice", "score", or "bool", got ${JSON.stringify(question.type)}`);
|
||||
}
|
||||
}
|
||||
return context as unknown as ClassifierContext;
|
||||
}
|
||||
|
||||
/** Check a script's image context, so mistakes such as `{ prompt }` fail with the expected shape. */
|
||||
function checkImagesContext(context: unknown): ImagesContext {
|
||||
const fail = (problem: string) =>
|
||||
new Error(
|
||||
`models.generateImages() ${problem}. Expected context: { input: [{ type: "text", text: <prompt> }, ...optional { type: "image", data: <base64>, mimeType } references] }. See "Generate images" in ${CODEMODE_DOCS_PATH}.`,
|
||||
);
|
||||
if (!isRecord(context)) throw fail(`expects a context object as its second argument, got ${describeValue(context)}`);
|
||||
const { input } = context;
|
||||
if (!Array.isArray(input) || input.length === 0) {
|
||||
throw fail(`context.input must be a non-empty array of blocks, got ${describeValue(input)}`);
|
||||
}
|
||||
input.forEach((block: unknown, index) => {
|
||||
if (isRecord(block) && block.type === "text" && typeof block.text === "string") return;
|
||||
if (
|
||||
isRecord(block) &&
|
||||
block.type === "image" &&
|
||||
typeof block.data === "string" &&
|
||||
typeof block.mimeType === "string"
|
||||
) {
|
||||
return;
|
||||
}
|
||||
throw fail(`context.input[${index}] must be a text or image block, got ${describeValue(block)}`);
|
||||
});
|
||||
return context as unknown as ImagesContext;
|
||||
}
|
||||
|
||||
/** The fields of `ClassifierResult` and `AssistantImages` that a nested call row reports. */
|
||||
interface ModelCallResult {
|
||||
stopReason: "stop" | "error" | "aborted";
|
||||
|
|
@ -243,6 +330,11 @@ export async function executeCodemode(
|
|||
const calls: CodemodeNestedCall[] = [];
|
||||
// Usage of the script's `models.*` calls. Nested tool calls report theirs through the session.
|
||||
let modelUsage: Usage | undefined;
|
||||
// Images returned by `models.generateImages()`, to notice a script that never shows them.
|
||||
let generatedImages = 0;
|
||||
const addGeneratedImages = (count: number) => {
|
||||
generatedImages += count;
|
||||
};
|
||||
const addModelUsage = (usage: Usage) => {
|
||||
modelUsage = modelUsage ? combineUsage(modelUsage, usage) : usage;
|
||||
};
|
||||
|
|
@ -287,7 +379,7 @@ export async function executeCodemode(
|
|||
globals: [
|
||||
...createDiscoveryGlobals(callable, samples, options),
|
||||
...(options.models && ctx
|
||||
? createModelGlobals(ctx.modelRegistry, toolCallId, calls, publish, addModelUsage)
|
||||
? createModelGlobals(ctx.modelRegistry, toolCallId, calls, publish, addModelUsage, addGeneratedImages)
|
||||
: []),
|
||||
],
|
||||
timeoutMs: sourceOptions.timeoutMs ?? Number.POSITIVE_INFINITY,
|
||||
|
|
@ -321,6 +413,12 @@ export async function executeCodemode(
|
|||
} else {
|
||||
items.push({ type: "text", text: `Script error:\n${formatError(result, calls)}` });
|
||||
}
|
||||
if (generatedImages > 0 && !items.some((item) => item.type === "image")) {
|
||||
items.push({
|
||||
type: "text",
|
||||
text: `Note: models.generateImages() returned ${generatedImages} image${generatedImages === 1 ? "" : "s"} that the script did not show. Show each image block of result.output with image(block).`,
|
||||
});
|
||||
}
|
||||
|
||||
const truncated = await truncateOutput(items, sourceOptions.maxOutputTokens ?? DEFAULT_MAX_OUTPUT_TOKENS);
|
||||
const wallTime = ((performance.now() - startedAt) / 1000).toFixed(1);
|
||||
|
|
@ -419,7 +517,7 @@ function createDiscoveryGlobals(
|
|||
}
|
||||
|
||||
/**
|
||||
* `models.*` for scripts: the model registry methods declared in {@link MODEL_GLOBAL_DECLARATIONS}.
|
||||
* `models.*` for scripts: the model registry methods documented in docs/codemode.md.
|
||||
* Classifier and image calls appear as nested call rows so the renderer shows them, and their usage
|
||||
* goes to `addUsage`. Rows show only the model, never prompts or image data.
|
||||
*/
|
||||
|
|
@ -429,26 +527,47 @@ function createModelGlobals(
|
|||
calls: CodemodeNestedCall[],
|
||||
publish: () => void,
|
||||
addUsage: (usage: Usage) => void,
|
||||
addGeneratedImages: (count: number) => void,
|
||||
): CodemodeTool[] {
|
||||
const limit = createLimiter(MAX_CONCURRENT_MODEL_CALLS);
|
||||
let callCount = 0;
|
||||
|
||||
/**
|
||||
* Resolve the script's model by provider and id only, then run the call as a nested call row. A
|
||||
* script-supplied baseUrl or headers must never receive the credentials.
|
||||
* Resolve the script's model by provider and id only, check the context, then run the call as a
|
||||
* nested call row. A script-supplied baseUrl or headers must never receive the credentials.
|
||||
*/
|
||||
const runModelCall = async <TType extends "classifier" | "image", TResult extends ModelCallResult>(
|
||||
const runModelCall = async <TType extends "classifier" | "image", TContext, TResult extends ModelCallResult>(
|
||||
name: string,
|
||||
type: TType,
|
||||
model: unknown,
|
||||
run: (resolved: ModelTypeMap[TType]) => Promise<TResult>,
|
||||
[model, context]: unknown[],
|
||||
checkContext: (context: unknown) => TContext,
|
||||
run: (resolved: ModelTypeMap[TType], context: TContext) => Promise<TResult>,
|
||||
): Promise<TResult> => {
|
||||
const ref = model as { provider?: unknown; id?: unknown } | null;
|
||||
if (typeof ref !== "object" || ref === null || typeof ref.provider !== "string" || typeof ref.id !== "string") {
|
||||
throw new Error(`${name}() expects a model from models.getModelOfType() or models.getAvailableOfType()`);
|
||||
const listHint = `List the ${type} models you can use with models.getAvailableOfType("${type}").`;
|
||||
if (!isRecord(model) || typeof model.provider !== "string" || typeof model.id !== "string") {
|
||||
// undefined arrives as null: spread arguments cross the sandbox as a JSON array.
|
||||
const undefinedHint =
|
||||
model === undefined || model === null
|
||||
? " models.getModelOfType() returns undefined for an unknown provider or id."
|
||||
: "";
|
||||
throw new Error(
|
||||
`${name}() expects ${withArticle(type)} model as its first argument, got ${describeValue(model)}.${undefinedHint} ${listHint}`,
|
||||
);
|
||||
}
|
||||
const resolved = models.getModelOfType(type, ref.provider, ref.id);
|
||||
if (!resolved) throw new Error(`Unknown ${type} model "${ref.provider}/${ref.id}"`);
|
||||
const { provider, id } = model;
|
||||
const ref = `${provider}/${id}`;
|
||||
const resolved = models.getModelOfType(type, provider, id);
|
||||
if (!resolved) {
|
||||
const actualType = [...MODEL_TYPES].find(
|
||||
(other) => other !== type && models.getModelOfType(other as ModelType, provider, id) !== undefined,
|
||||
);
|
||||
throw new Error(
|
||||
actualType
|
||||
? `"${ref}" is ${withArticle(actualType)} model, not ${withArticle(type)} model. ${listHint}`
|
||||
: `Unknown ${type} model "${ref}". ${listHint}`,
|
||||
);
|
||||
}
|
||||
const checked = checkContext(context);
|
||||
|
||||
const record: CodemodeNestedCall = {
|
||||
id: `${toolCallId}/${name}/${++callCount}`,
|
||||
|
|
@ -459,7 +578,7 @@ function createModelGlobals(
|
|||
calls.push(record);
|
||||
publish();
|
||||
const startedAt = performance.now();
|
||||
const result = await limit(() => run(resolved));
|
||||
const result = await limit(() => run(resolved, checked));
|
||||
record.durationMs = performance.now() - startedAt;
|
||||
record.status = result.stopReason === "stop" ? "ok" : result.stopReason === "aborted" ? "cancelled" : "error";
|
||||
if (result.errorMessage) record.error = truncateText(result.errorMessage, ERROR_PREVIEW_CHARS);
|
||||
|
|
@ -483,27 +602,29 @@ function createModelGlobals(
|
|||
"models.getModelOfType": (args) => {
|
||||
const [type, provider, id] = args as unknown[];
|
||||
if (typeof provider !== "string" || typeof id !== "string") {
|
||||
throw new Error("models.getModelOfType() expects a type, a provider, and an id");
|
||||
throw new Error(
|
||||
`models.getModelOfType(type, provider, id) expects three strings, got (${(args as unknown[]).map(describeValue).join(", ")}). The provider and the id are separate arguments, for example models.getModelOfType("classifier", "typesafe", "jev-latest").`,
|
||||
);
|
||||
}
|
||||
const model = models.getModelOfType(toModelType(type), provider, id);
|
||||
return model === undefined ? undefined : toModelInfo(model);
|
||||
},
|
||||
"models.classify": (args, { signal }) => {
|
||||
const [model, context] = args as unknown[];
|
||||
return runModelCall("models.classify", "classifier", model, (resolved) =>
|
||||
models.classify(resolved, context as ClassifierContext, { signal }),
|
||||
);
|
||||
},
|
||||
"models.generateImages": (args, { signal }) => {
|
||||
const [model, context] = args as unknown[];
|
||||
return runModelCall("models.generateImages", "image", model, (resolved) =>
|
||||
models.generateImages(resolved, context as ImagesContext, { signal }),
|
||||
);
|
||||
},
|
||||
"models.classify": (args, { signal }) =>
|
||||
runModelCall("models.classify", "classifier", args as unknown[], checkClassifierContext, (resolved, context) =>
|
||||
models.classify(resolved, context, { signal }),
|
||||
),
|
||||
"models.generateImages": (args, { signal }) =>
|
||||
runModelCall(
|
||||
"models.generateImages",
|
||||
"image",
|
||||
args as unknown[],
|
||||
checkImagesContext,
|
||||
async (resolved, context) => {
|
||||
const result = await models.generateImages(resolved, context, { signal });
|
||||
addGeneratedImages(result.output.filter((block) => block.type === "image").length);
|
||||
return result;
|
||||
},
|
||||
),
|
||||
};
|
||||
return MODEL_GLOBAL_DECLARATIONS.map((declaration) => ({
|
||||
name: declaration.name,
|
||||
spread: true,
|
||||
execute: implementations[declaration.name],
|
||||
}));
|
||||
return Object.entries(implementations).map(([name, execute]) => ({ name, spread: true, execute }));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,17 +21,19 @@
|
|||
* so each branch sees the values written on its own path.
|
||||
*/
|
||||
|
||||
import { join } from "node:path";
|
||||
import type { AgentTool } from "@earendil-works/pi-agent-core";
|
||||
import type { CodemodeJsonSchema, CodemodeTool } from "@earendil-works/pi-codemode";
|
||||
import {
|
||||
MCP_TYPESCRIPT_PREAMBLE,
|
||||
mcpStructuredContentSchema,
|
||||
renderDeclarations,
|
||||
renderToolOutputType,
|
||||
renderToolSample,
|
||||
toCodemodeIdentifier,
|
||||
} from "@earendil-works/pi-codemode/declarations";
|
||||
import { CODEMODE_SOURCE_GRAMMAR } from "@earendil-works/pi-codemode/source";
|
||||
import { type Static, Type } from "typebox";
|
||||
import { getDocsPath } from "../../config.ts";
|
||||
import type {
|
||||
ToolDefinition,
|
||||
ToolInfo,
|
||||
|
|
@ -84,8 +86,7 @@ const TEXT_OUTPUT_SCHEMA: CodemodeJsonSchema = { type: "string" };
|
|||
|
||||
export const codemodeSchema = Type.Object({
|
||||
code: Type.String({
|
||||
description:
|
||||
'Raw JavaScript source. Top-level await and return work. May start with a `// @options: {"max_output_tokens": 1000}` line.',
|
||||
description: "Raw JavaScript source.",
|
||||
}),
|
||||
});
|
||||
|
||||
|
|
@ -122,128 +123,32 @@ export interface CodemodeToolDetails {
|
|||
}
|
||||
|
||||
export const codemodeToolSystemPromptContribution = {
|
||||
snippet: "Run JavaScript that calls other tools (chains, loops, Promise.all, filtering large results)",
|
||||
snippet: "Run JavaScript that calls other tools",
|
||||
guidelines: [
|
||||
"Use codemode to batch or chain several tool calls, or to filter large tool output down to what you need, instead of issuing many individual tool calls. Batch independent calls in one codemode call using await Promise.allSettled([...]).",
|
||||
"Use codemode to batch independent tool calls (Promise.allSettled), chain them, or filter large output, instead of many separate calls.",
|
||||
],
|
||||
} as const;
|
||||
|
||||
const DESCRIPTION_INTRO = `Run JavaScript code to orchestrate/compose tool calls
|
||||
- Evaluates the provided JavaScript code in a fresh QuickJS sandbox as the body of an async function: top-level \`await\` and \`return\` work.
|
||||
- All nested tools are available on the global \`tools\` object, for example \`await tools.read(...)\`. Tool names are exposed as normalized JavaScript identifiers, for example \`await tools.mcp__ologs__get_profile(...)\`.
|
||||
- Nested tool methods take an object as their input argument.
|
||||
- Nested tools return either an object or a string, based on the description.
|
||||
- A nested tool call that fails, is blocked, or gets invalid arguments rejects with an Error carrying the tool's error text.
|
||||
- Runs raw JavaScript -- no Node, no file system, no network access, no timers.
|
||||
- Accepts raw JavaScript source text, not JSON, quoted strings, or markdown code fences.
|
||||
- You may optionally start the tool input with a first line like \`// @options: {"max_output_tokens": 1000, "timeout_ms": 60000}\`.
|
||||
- \`max_output_tokens\` sets the token budget for the script's output. Defaults to 10000 tokens.
|
||||
- \`timeout_ms\` sets a hard deadline for the whole script. By default there is none.
|
||||
- When the JS code is fully evaluated, calls that are still running are cancelled and unawaited promises are silently discarded.
|
||||
- Tool calls are real and have side effects. If the script fails partway, earlier calls are not undone.
|
||||
- Scripts have a 256 MB memory limit; exceeding it throws \`InternalError: out of memory\`. Filter or aggregate large data instead of accumulating it.
|
||||
/** The reference for scripts: globals, tool results, `store()`, the `models` API, and limits. */
|
||||
export const CODEMODE_DOCS_PATH = join(getDocsPath(), "codemode.md");
|
||||
|
||||
- Global helpers:
|
||||
- \`exit()\`: Immediately ends the current script successfully (like an early return from the top level).
|
||||
- \`text(value: string | number | boolean | undefined | null)\`: Appends a text item. Non-string values are stringified with \`JSON.stringify(...)\` when possible.
|
||||
- \`image(imageUrlOrItem: string | { image_url: string } | ImageContent)\`: Appends an image item. \`image_url\` should be a base64-encoded \`data:\` URL. To forward an MCP tool image, pass an individual \`ImageContent\` block from \`result.content\`, for example \`image(result.content[0])\`.
|
||||
- \`store(key: string, value: any)\`: stores a serializable value under a string key for later \`codemode\` calls in the same session. Storing \`undefined\` deletes the key. Writes are kept only if the script succeeds.
|
||||
- \`load(key: string)\`: returns the stored value for a string key, or \`undefined\` if it is missing.
|
||||
- \`ALL_TOOLS\`: metadata for the enabled nested tools as \`{ name, description }\` entries.
|
||||
- \`searchTools(query: string, options?: { limit?: number; namespace?: string })\`: resolves to the nested tools that best match the query (BM25, default limit 8), as \`{ name, description }\` entries like \`ALL_TOOLS\`.
|
||||
- \`describeTool(name: string)\`: resolves to the description and declaration of a nested tool, or \`undefined\`.
|
||||
- \`describeNamespace(name: string)\`: resolves to \`{ name, description?, instructions?, tools }\` for a namespace of nested tools, such as an MCP server: its usage instructions and the names of its tools, or \`undefined\`.
|
||||
- \`console.log(...)\` and the other \`console\` methods append a text item like \`text()\`.
|
||||
- \`return value\` at the top level appends the value like \`text()\`.`;
|
||||
const DESCRIPTION_INTRO = `Run JavaScript that calls other tools. The input is raw JavaScript (not JSON, no code fence), run as an async function body in a QuickJS sandbox: top-level \`await\` and \`return\` work. No Node, file system, network, or timers.
|
||||
- \`await tools.<name>({ ...args })\` resolves to a string, or an object if the tool's declaration says so, and rejects with an Error on failure. Calls still running when the script ends are cancelled.
|
||||
- Optional first line: \`// @options: {"max_output_tokens": 10000, "timeout_ms": 60000}\``;
|
||||
|
||||
const MODEL_TYPES = `type ModelType = "chat" | "image" | "classifier";
|
||||
/** A model catalog entry. \`provider\` and \`id\` identify it; the other fields depend on the type. */
|
||||
interface ModelInfo {
|
||||
type?: ModelType;
|
||||
provider: string;
|
||||
id: string;
|
||||
name: string;
|
||||
api: string;
|
||||
input: ("text" | "image")[];
|
||||
contextWindow?: number;
|
||||
[key: string]: unknown;
|
||||
/** One line per global. The details live in {@link CODEMODE_DOCS_PATH}. */
|
||||
function describeGlobals(models: boolean): string {
|
||||
const lines = [
|
||||
"Globals:",
|
||||
"- `text(value)`, `image(dataUrlOrImageBlock)`, `console.log(...)`, and top-level `return` add output; `exit()` ends the script.",
|
||||
"- `store(key, value)` and `load(key)` keep JSON values across codemode calls.",
|
||||
"- `ALL_TOOLS`, `searchTools(query, { limit?, namespace? })`, `describeTool(name)`, `describeNamespace(name)`: find unlisted tools, such as MCP tools.",
|
||||
];
|
||||
if (models) {
|
||||
lines.push(`- \`models\`: classifiers and image generation. Read ${CODEMODE_DOCS_PATH} first.`);
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
type ClassifierQuestion =
|
||||
| { type: "choice"; instructions: string; criteria: Record<string, string> }
|
||||
| { type: "score"; instructions: string; criteria: string[] }
|
||||
| { type: "bool"; instructions: string; criteria: { true: string; false: string } };
|
||||
type ClassifierAnswer =
|
||||
| { type: "choice"; choice: string; probabilities: Record<string, number>; confidence: number }
|
||||
| { type: "score"; score: number; confidence: number }
|
||||
| { type: "bool"; probability: number };
|
||||
interface ClassifierContext {
|
||||
state: Record<string, unknown>;
|
||||
questions: Record<string, ClassifierQuestion>;
|
||||
}
|
||||
/** Token counts reported by the service. Cost is in USD. */
|
||||
type ModelUsage = { input: number; output: number; totalTokens: number; cost: { total: number } };
|
||||
interface ClassifierResult {
|
||||
api: string;
|
||||
provider: string;
|
||||
model: string;
|
||||
answers: Record<string, ClassifierAnswer>;
|
||||
usage?: ModelUsage;
|
||||
stopReason: "stop" | "error" | "aborted";
|
||||
errorMessage?: string;
|
||||
timestamp: number;
|
||||
}
|
||||
type ModelTextBlock = { type: "text"; text: string };
|
||||
/** \`data\` is base64. Show it with \`image(block)\`; never print \`data\` with \`text()\`, \`console\`, or \`return\`. */
|
||||
type ModelImageBlock = { type: "image"; data: string; mimeType: string };
|
||||
interface ImagesContext {
|
||||
/** The prompt as text blocks, plus image blocks to edit or use as references. */
|
||||
input: (ModelTextBlock | ModelImageBlock)[];
|
||||
}
|
||||
interface ImagesResult {
|
||||
api: string;
|
||||
provider: string;
|
||||
model: string;
|
||||
/** Generated images, and text blocks for models that also return text. */
|
||||
output: (ModelTextBlock | ModelImageBlock)[];
|
||||
usage?: ModelUsage;
|
||||
stopReason: "stop" | "error" | "aborted";
|
||||
errorMessage?: string;
|
||||
timestamp: number;
|
||||
}`;
|
||||
|
||||
/** Declarations of the `models` globals; codemode-execute.ts implements them. */
|
||||
export const MODEL_GLOBAL_DECLARATIONS: readonly Omit<CodemodeTool, "execute">[] = [
|
||||
{
|
||||
name: "models.getModelsOfType",
|
||||
description: "Every known model of a type, optionally for one provider.",
|
||||
signature: "(type: ModelType, provider?: string): Promise<ModelInfo[]>",
|
||||
},
|
||||
{
|
||||
name: "models.getAvailableOfType",
|
||||
description: "Models of a type whose provider has working credentials.",
|
||||
signature: "(type: ModelType, provider?: string): Promise<ModelInfo[]>",
|
||||
},
|
||||
{
|
||||
name: "models.getModelOfType",
|
||||
description: "One catalog entry, or undefined.",
|
||||
signature: "(type: ModelType, provider: string, id: string): Promise<ModelInfo | undefined>",
|
||||
},
|
||||
{
|
||||
name: "models.classify",
|
||||
description:
|
||||
"Run a classifier model on one state. Only `provider` and `id` of `model` are used. Provider errors do not throw: check `stopReason` and `errorMessage`.",
|
||||
signature: "(model: ModelInfo, context: ClassifierContext): Promise<ClassifierResult>",
|
||||
},
|
||||
{
|
||||
name: "models.generateImages",
|
||||
description:
|
||||
'Generate images with an image model. Only `provider` and `id` of `model` are used. Provider errors do not throw: check `stopReason` and `errorMessage`. Generation can take minutes, so do not set a short `timeout_ms`. Show results with `for (const block of result.output) if (block.type === "image") image(block);`.',
|
||||
signature: "(model: ModelInfo, context: ImagesContext): Promise<ImagesResult>",
|
||||
},
|
||||
];
|
||||
|
||||
const DEFERRED_TOOLS_GUIDANCE = `Some nested tools may be omitted from this description, such as deferred tools and MCP tools. They are still available on the global \`tools\` object and listed in \`ALL_TOOLS\`.
|
||||
To find one, call \`await searchTools(query)\` (pass \`{ namespace }\` to search one namespace), or filter \`ALL_TOOLS\` by \`name\` and \`description\`. \`await describeNamespace(name)\` returns a namespace's usage instructions and the names of its tools.`;
|
||||
|
||||
/** Default for {@link CodemodeDescriptionOptions.inlineBudget}, in estimated tokens. */
|
||||
export const DEFAULT_CODEMODE_INLINE_BUDGET = 3000;
|
||||
|
|
@ -350,7 +255,7 @@ export function createCodemodeDescription(
|
|||
);
|
||||
const shown = selectCatalog(ordered, options.inlineBudget);
|
||||
|
||||
const sections = [DESCRIPTION_INTRO, DEFERRED_TOOLS_GUIDANCE];
|
||||
const sections = [DESCRIPTION_INTRO, describeGlobals(options.models === true)];
|
||||
if (
|
||||
declarations.some(
|
||||
(declaration) =>
|
||||
|
|
@ -359,13 +264,6 @@ export function createCodemodeDescription(
|
|||
) {
|
||||
sections.push(`Shared MCP Types:\n\`\`\`ts\n${MCP_TYPESCRIPT_PREAMBLE}\n\`\`\``);
|
||||
}
|
||||
if (options.models) {
|
||||
const noop = () => undefined;
|
||||
const models = renderDeclarations({
|
||||
globals: MODEL_GLOBAL_DECLARATIONS.map((global) => ({ ...global, execute: noop })),
|
||||
});
|
||||
sections.push(`Model API:\n\`\`\`ts\n${MODEL_TYPES}\n\n${models}\n\`\`\``);
|
||||
}
|
||||
if (declarations.length === 0) return sections.join("\n\n");
|
||||
|
||||
const toolSections = ["Nested tools:"];
|
||||
|
|
@ -388,9 +286,39 @@ export function createCodemodeDescription(
|
|||
return sections.join("\n\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* What a script call resolves to, in one line: `a string`, the field names of an object
|
||||
* (`{ output, exit_code, full_output_path? }`), or the rendered type for anything else.
|
||||
*/
|
||||
function describeOutput(schema: CodemodeJsonSchema | undefined): string {
|
||||
const type = renderToolOutputType(schema);
|
||||
if (type === "string") return "a string";
|
||||
const object = typeof schema === "object" ? schema : undefined;
|
||||
const properties = object?.properties;
|
||||
if (
|
||||
object?.type === "object" &&
|
||||
typeof properties === "object" &&
|
||||
properties !== null &&
|
||||
mcpStructuredContentSchema(schema) === undefined
|
||||
) {
|
||||
const required = new Set(Array.isArray(object.required) ? object.required : []);
|
||||
const fields = Object.keys(properties).map((name) => (required.has(name) ? name : `${name}?`));
|
||||
return `\`{ ${fields.join(", ")} }\``;
|
||||
}
|
||||
return `\`${type.replace(/\s+/g, " ")}\``;
|
||||
}
|
||||
|
||||
/**
|
||||
* A declared tool's description followed by how scripts call it and what the call resolves to. The
|
||||
* arguments are the tool's declared parameters, so they are not repeated.
|
||||
*/
|
||||
function describeScriptCall(tool: AgentTool<any>): string {
|
||||
return `${tool.description.trim()}\n\nCodemode: \`tools.${toCodemodeIdentifier(tool.name)}(args)\` resolves to ${describeOutput(toCodemodeDeclaration(tool).outputSchema)}.`;
|
||||
}
|
||||
|
||||
/**
|
||||
* How the codemode tool presents tools that are both declared and callable from scripts:
|
||||
* - `on`: their descriptions get the codemode declaration appended, and the codemode description
|
||||
* - `on`: their descriptions say how scripts call them, and the codemode description
|
||||
* lists only the callable tools without `direct` exposure.
|
||||
* - `only`: the codemode description lists every callable tool, and requests leave out the
|
||||
* declarations of active `direct` tools.
|
||||
|
|
@ -406,7 +334,7 @@ function prepareCodemodeLoadout(loadout: ToolLoadout, options: CodemodeToolOptio
|
|||
const descriptions: Record<string, string> = {};
|
||||
if (mode === "on") {
|
||||
for (const tool of loadout.declared) {
|
||||
if (callableNames.has(tool.name)) descriptions[tool.name] = renderToolSample(toCodemodeDeclaration(tool));
|
||||
if (callableNames.has(tool.name)) descriptions[tool.name] = describeScriptCall(tool);
|
||||
}
|
||||
}
|
||||
const listed = mode === "only" ? callable : callable.filter((tool) => !isDirect(tool));
|
||||
|
|
|
|||
|
|
@ -155,8 +155,13 @@ const MAX_SERVER_DESCRIPTION_CHARS = 250;
|
|||
*/
|
||||
export const MAX_SERVERS_SECTION_CHARS = 4096;
|
||||
|
||||
const SERVERS_SECTION_INTRO =
|
||||
"MCP servers whose tools are not declared to you. Call the tools of `codemode` servers from codemode scripts: find them with `searchTools(query, { namespace })` and read a server's instructions and tool names with `describeNamespace(name)`. Load the tools of `tool_search` servers with `tool_search`.";
|
||||
/** The section's first line. It explains only the ways of reaching tools that the listed servers use. */
|
||||
function serversSectionIntro(reaches: ReadonlySet<string>): string {
|
||||
let intro = "MCP servers whose tools are not declared to you.";
|
||||
if (reaches.has("codemode")) intro += " Call the tools of `codemode` servers from codemode scripts.";
|
||||
if (reaches.has("tool_search")) intro += " Load the tools of `tool_search` servers with `tool_search`.";
|
||||
return intro;
|
||||
}
|
||||
|
||||
function truncate(text: string, max: number): string {
|
||||
if (text.length <= max) return text;
|
||||
|
|
@ -185,16 +190,15 @@ export function renderServersSection(servers: readonly McpServerListing[]): stri
|
|||
.filter((server) => isEnabled(server) && hasIndirectTools(server.entry))
|
||||
.sort((a, b) => a.entry.name.localeCompare(b.entry.name));
|
||||
if (listed.length === 0) return undefined;
|
||||
const heads = listed.map((server) => {
|
||||
const exposures = configuredExposures(server.entry);
|
||||
const reach = exposures.has("codemode") ? "codemode" : "tool_search";
|
||||
return `- ${mcpNamespace(server.entry.name)} (${reach})`;
|
||||
});
|
||||
const reaches = listed.map((server) =>
|
||||
configuredExposures(server.entry).has("codemode") ? "codemode" : "tool_search",
|
||||
);
|
||||
const intro = serversSectionIntro(new Set(reaches));
|
||||
const heads = listed.map((server, index) => `- ${mcpNamespace(server.entry.name)} (${reaches[index]})`);
|
||||
const omitted = (count: number) =>
|
||||
count > 0 ? [`- … ${count} more server${count === 1 ? "" : "s"}; find their tools with searchTools()`] : [];
|
||||
// Characters of the intro, the first `kept` server lines without descriptions, and the omission line.
|
||||
const size = (kept: number) =>
|
||||
[SERVERS_SECTION_INTRO, ...heads.slice(0, kept), ...omitted(listed.length - kept)].join("\n").length;
|
||||
const size = (kept: number) => [intro, ...heads.slice(0, kept), ...omitted(listed.length - kept)].join("\n").length;
|
||||
let kept = listed.length;
|
||||
while (kept > 0 && size(kept) > MAX_SERVERS_SECTION_CHARS) kept--;
|
||||
// Each description also takes a ": " separator.
|
||||
|
|
@ -206,7 +210,7 @@ export function renderServersSection(servers: readonly McpServerListing[]): stri
|
|||
const summary = perServer > 0 ? truncate(serverSummary(server), perServer) : "";
|
||||
return summary ? `${heads[index]}: ${summary}` : heads[index];
|
||||
});
|
||||
return [SERVERS_SECTION_INTRO, ...lines, ...omitted(listed.length - kept)].join("\n");
|
||||
return [intro, ...lines, ...omitted(listed.length - kept)].join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import { createToolDefinitionFromAgentTool } from "../../src/core/tools/tool-def
|
|||
import { readCodemodeStore } from "../../src/extensions/codemode/execute.ts";
|
||||
import { createCodemodeExtension } from "../../src/extensions/codemode/index.ts";
|
||||
import {
|
||||
CODEMODE_DOCS_PATH,
|
||||
CODEMODE_STORE_ENTRY_TYPE,
|
||||
type CodemodeToolDetails,
|
||||
createCodemodeTool,
|
||||
|
|
@ -126,9 +127,10 @@ describe("AgentSession codemode tool", () => {
|
|||
return fauxAssistantMessage("ok");
|
||||
};
|
||||
|
||||
// on: declared tools carry their codemode declaration and are not listed again in codemode.
|
||||
// on: declared tools say how scripts call them and are not listed again in codemode.
|
||||
harness.session.setActiveToolsByName(["read", "echo", "codemode"]);
|
||||
expect(description("echo")).toContain("codemode tool declaration:");
|
||||
expect(description("echo")).toContain("Codemode: `tools.echo(args)` resolves to");
|
||||
expect(description("echo")).not.toContain("codemode tool declaration:");
|
||||
expect(description("codemode")).not.toContain("### `echo`");
|
||||
harness.setResponses([record]);
|
||||
await harness.session.prompt("on");
|
||||
|
|
@ -138,7 +140,7 @@ describe("AgentSession codemode tool", () => {
|
|||
// only: codemode lists echo, which stays active but is left out of requests.
|
||||
harness.settingsManager.applyOverrides({ codemode: { mode: "only" } });
|
||||
harness.session.setActiveToolsByName(["read", "echo", "codemode"]);
|
||||
expect(description("echo")).not.toContain("codemode tool declaration:");
|
||||
expect(description("echo")).not.toContain("Codemode: `tools.echo");
|
||||
expect(description("codemode")).toContain("### `echo`");
|
||||
expect(description("codemode")).not.toContain("### `stats`");
|
||||
harness.setResponses([record]);
|
||||
|
|
@ -660,20 +662,14 @@ describe("codemode models", () => {
|
|||
it("declares models only for the session's own codemode tool", async () => {
|
||||
const { harness } = await setup();
|
||||
const codemode = harness.session.agent.state.tools.find((tool) => tool.name === "codemode");
|
||||
expect(codemode?.description).toContain("declare const models: {");
|
||||
expect(codemode?.description).toContain(
|
||||
"classify(model: ModelInfo, context: ClassifierContext): Promise<ClassifierResult>;",
|
||||
);
|
||||
expect(codemode?.description).toContain("interface ClassifierResult {");
|
||||
expect(codemode?.description).toContain(
|
||||
"generateImages(model: ModelInfo, context: ImagesContext): Promise<ImagesResult>;",
|
||||
);
|
||||
expect(codemode?.description).toContain("interface ImagesResult {");
|
||||
// The description names the models globals and points to the docs for the API.
|
||||
expect(codemode?.description).toContain("`models`: classifiers and image generation");
|
||||
expect(codemode?.description).toContain(CODEMODE_DOCS_PATH);
|
||||
|
||||
const overridden = await createHarness({ tools: [createCodemodeTool() as AgentTool] });
|
||||
harnesses.push(overridden);
|
||||
const plain = overridden.session.agent.state.tools.find((tool) => tool.name === "codemode");
|
||||
expect(plain?.description).not.toContain("declare const models");
|
||||
expect(plain?.description).not.toContain("`models`");
|
||||
});
|
||||
|
||||
it("lists models and classifies with catalog auth, ignoring script-supplied fields", async () => {
|
||||
|
|
@ -758,7 +754,8 @@ describe("codemode models", () => {
|
|||
id: "painter",
|
||||
stopReason: "stop",
|
||||
failed: ["error", "painter exploded"],
|
||||
wrongType: 'Unknown image model "scorer/judge"',
|
||||
wrongType:
|
||||
'"scorer/judge" is a classifier model, not an image model. List the image models you can use with models.getAvailableOfType("image").',
|
||||
});
|
||||
expect(result.content[2]).toEqual({ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" });
|
||||
expect(imageRequests.map((request) => [request.baseUrl, request.apiKey])).toEqual([
|
||||
|
|
@ -778,6 +775,22 @@ describe("codemode models", () => {
|
|||
expect(harness.session.getSessionStats().cost).toBeCloseTo(0.04, 10);
|
||||
});
|
||||
|
||||
it("notes generated images that the script did not show", async () => {
|
||||
const { harness } = await setup();
|
||||
const result = await run(
|
||||
harness,
|
||||
`
|
||||
const [model] = await models.getAvailableOfType("image", "scorer");
|
||||
const generated = await models.generateImages(model, { input: [{ type: "text", text: "a fox" }] });
|
||||
return generated.stopReason;
|
||||
`,
|
||||
);
|
||||
expect(result.isError).toBe(false);
|
||||
expect(resultText(result)).toBe(
|
||||
"stop\nNote: models.generateImages() returned 1 image that the script did not show. Show each image block of result.output with image(block).",
|
||||
);
|
||||
});
|
||||
|
||||
it("reports provider errors as results and invalid arguments as exceptions", async () => {
|
||||
const { harness } = await setup();
|
||||
const result = await run(
|
||||
|
|
@ -791,6 +804,13 @@ describe("codemode models", () => {
|
|||
badType: await attempt(() => models.getModelsOfType("video")),
|
||||
unknown: await attempt(() => models.classify({ provider: "scorer", id: "nope" }, {})),
|
||||
noModel: await attempt(() => models.classify("judge", {})),
|
||||
undefinedModel: await attempt(() => models.classify(undefined, {})),
|
||||
noState: await attempt(() => models.classify(model, { questions: ${questions} })),
|
||||
badQuestion: await attempt(() =>
|
||||
models.classify(model, { state: {}, questions: { kind: { type: "choice", instructions: "Kind?", criteria: ["a", "b"] } } }),
|
||||
),
|
||||
badImage: await attempt(() => models.generateImages({ provider: "scorer", id: "painter" }, { prompt: "a fox" })),
|
||||
badSplit: await attempt(() => models.getModelOfType("classifier", "scorer/judge")),
|
||||
};
|
||||
`,
|
||||
);
|
||||
|
|
@ -798,8 +818,24 @@ describe("codemode models", () => {
|
|||
const value = JSON.parse(resultText(result));
|
||||
expect(value.failed).toEqual(["error", "classifier exploded"]);
|
||||
expect(value.badType).toContain('Unknown model type "video"');
|
||||
expect(value.unknown).toBe('Unknown classifier model "scorer/nope"');
|
||||
expect(value.noModel).toContain("expects a model");
|
||||
expect(value.unknown).toBe(
|
||||
'Unknown classifier model "scorer/nope". List the classifier models you can use with models.getAvailableOfType("classifier").',
|
||||
);
|
||||
expect(value.noModel).toContain(
|
||||
"models.classify() expects a classifier model as its first argument, got a string.",
|
||||
);
|
||||
expect(value.undefinedModel).toContain(
|
||||
"models.getModelOfType() returns undefined for an unknown provider or id.",
|
||||
);
|
||||
expect(value.noState).toContain("models.classify() context.state must be an object, got undefined.");
|
||||
expect(value.noState).toContain("codemode.md");
|
||||
expect(value.badQuestion).toContain(
|
||||
'context.questions.kind is a "choice" question, so criteria must map each label to its meaning.',
|
||||
);
|
||||
expect(value.badImage).toContain(
|
||||
"models.generateImages() context.input must be a non-empty array of blocks, got undefined.",
|
||||
);
|
||||
expect(value.badSplit).toContain("The provider and the id are separate arguments");
|
||||
const details = result.details as unknown as CodemodeToolDetails;
|
||||
expect(details.calls.map((call) => [call.name, call.status, call.error])).toEqual([
|
||||
["models.classify", "error", "classifier exploded"],
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ describe("codemode description catalog", () => {
|
|||
expect(description).toContain("## mcp__github\nGitHub server");
|
||||
expect(description).toContain("## mcp__docs\n\n### `mcp__docs");
|
||||
// The search guidance is always there, so tools that appear later do not change it.
|
||||
expect(description).toContain("To find one, call `await searchTools(query)`");
|
||||
expect(description).toContain("find unlisted tools, such as MCP tools");
|
||||
});
|
||||
|
||||
it("fills the budget round-robin, cheapest first, and says what is missing", () => {
|
||||
|
|
@ -91,7 +91,7 @@ describe("codemode description catalog", () => {
|
|||
expect(description).toContain("### `mcp__docs__search`");
|
||||
expect(description).not.toContain("### `mcp__docs__long`");
|
||||
expect(description).toContain("## mcp__github (some tools not listed)");
|
||||
expect(description).toContain("To find one, call `await searchTools(query)`");
|
||||
expect(description).toContain("find unlisted tools, such as MCP tools");
|
||||
// Deterministic: the same input gives the same description.
|
||||
expect(createCodemodeDescription(all, { namespaces, inlineBudget: 170 })).toBe(description);
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue