sscoble/pi-mono
1
0
Fork 0
mirror of https://github.com/badlogic/pi-mono.git synced 2026-11-05 23:37:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

fix(ai): keep Copilot tests aligned with current model catalog

Browse source 
Derive catalog-sensitive test IDs from generated models and update tool-call normalization coverage to GPT-5.5.
This commit is contained in:
David Brailovsky 2026-09-04 11:15:20 +02:00
parent 2d41163332
commit ef41a24b39
2 changed files with 51 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

67
packages/ai/test/github-copilot-oauth.test.ts
View file

@ -32,6 +32,12 @@ function getUrl(input: unknown): string {
throw new Error(`Unsupported fetch input: ${String(input)}`);
}
function requireModelId(models: readonly { id: string }[], index: number): string {
const model = models[index];
if (!model) throw new Error(`Expected a GitHub Copilot model at index ${index}`);
return model.id;
}
function stubGitHubCopilotLoginFetch(options: {
models: () => Response;
policy?: (modelId: string) => Response;
@ -135,68 +141,76 @@ describe("GitHub Copilot OAuth device flow", () => {
});
it("filters models to the authenticated account picker catalog", async () => {
const provider = githubCopilotProvider();
const providerModels = provider.getModels();
const pickerModelId = requireModelId(providerModels, 0);
const disabledModelId = requireModelId(providerModels, 1);
const hiddenModelId = requireModelId(providerModels, 2);
const credentials = await refreshGitHubCopilotModelsForTest([
{
id: "gpt-4.1",
id: pickerModelId,
model_picker_enabled: true,
capabilities: { supports: { tool_calls: true } },
},
{
id: "claude-opus-4.7",
id: disabledModelId,
model_picker_enabled: true,
policy: { state: "disabled" },
capabilities: { supports: { tool_calls: true } },
},
{
id: "gpt-5.4-nano",
id: hiddenModelId,
model_picker_enabled: false,
policy: { state: "enabled" },
capabilities: { supports: { tool_calls: true } },
},
]);
expect(credentials.availableModelIds).toEqual(["gpt-4.1"]);
expect(credentials.availableModelIds).toEqual([pickerModelId]);
const store = new InMemoryCredentialStore();
await store.modify("github-copilot", async () => ({ ...credentials, type: "oauth" }));
const models = createModels({ credentials: store });
models.setProvider(githubCopilotProvider());
expect((await models.getAvailable("github-copilot")).map((model) => model.id)).toEqual(["gpt-4.1"]);
models.setProvider(provider);
expect((await models.getAvailable("github-copilot")).map((model) => model.id)).toEqual([pickerModelId]);
});
it("falls back to explicitly enabled policy models when the picker catalog is empty", async () => {
const provider = githubCopilotProvider();
const enabledModelId = requireModelId(provider.getModels(), 0);
const credentials = await refreshGitHubCopilotModelsForTest([
{
id: "gpt-4.1",
id: enabledModelId,
model_picker_enabled: false,
policy: { state: "enabled" },
capabilities: { supports: { tool_calls: true } },
},
{
id: "claude-opus-4.7",
id: "policy-disabled-model",
model_picker_enabled: false,
policy: { state: "disabled" },
capabilities: { supports: { tool_calls: true } },
},
{
id: "gpt-5.4-nano",
id: "unconfigured-model",
model_picker_enabled: false,
capabilities: { supports: { tool_calls: true } },
},
{
id: "gpt-4o",
id: "tool-incapable-model",
model_picker_enabled: false,
policy: { state: "enabled" },
capabilities: { supports: { tool_calls: false } },
},
]);
expect(credentials.availableModelIds).toEqual(["gpt-4.1"]);
expect(credentials.availableModelIds).toEqual([enabledModelId]);
const store = new InMemoryCredentialStore();
await store.modify("github-copilot", async () => ({ ...credentials, type: "oauth" }));
const models = createModels({ credentials: store });
models.setProvider(githubCopilotProvider());
expect((await models.getAvailable("github-copilot")).map((model) => model.id)).toEqual(["gpt-4.1"]);
models.setProvider(provider);
expect((await models.getAvailable("github-copilot")).map((model) => model.id)).toEqual([enabledModelId]);
});
it("does not fall back to policy models for non-Individual accounts", async () => {
@ -308,6 +322,10 @@ describe("GitHub Copilot OAuth device flow", () => {
it("updates only known, tool-capable, unconfigured account model policies", async () => {
vi.useFakeTimers();
const providerModels = githubCopilotProvider().getModels();
const configuredModelId = requireModelId(providerModels, 0);
const unconfiguredModelId = requireModelId(providerModels, 1);
const toolIncapableModelId = requireModelId(providerModels, 2);
let catalogRequestCount = 0;
const policyModelIds: string[] = [];
stubGitHubCopilotLoginFetch({
@ -316,13 +334,13 @@ describe("GitHub Copilot OAuth device flow", () => {
return jsonResponse({
data: [
{
id: "gpt-4.1",
id: configuredModelId,
model_picker_enabled: true,
policy: { state: "enabled" },
capabilities: { supports: { tool_calls: true } },
},
{
id: "claude-sonnet-4.5",
id: unconfiguredModelId,
model_picker_enabled: true,
policy: { state: "unconfigured" },
capabilities: { supports: { tool_calls: true } },
@ -334,7 +352,7 @@ describe("GitHub Copilot OAuth device flow", () => {
capabilities: { supports: { tool_calls: true } },
},
{
id: "gpt-5.4",
id: toolIncapableModelId,
model_picker_enabled: true,
policy: { state: "unconfigured" },
capabilities: { supports: { tool_calls: false } },
@ -356,17 +374,18 @@ describe("GitHub Copilot OAuth device flow", () => {
await loginPromise;
expect(catalogRequestCount).toBe(1);
expect(policyModelIds).toEqual(["claude-sonnet-4.5"]);
expect(policyModelIds).toEqual([unconfiguredModelId]);
});
it("retries a throttled policy update after Retry-After", async () => {
vi.useFakeTimers();
const modelId = requireModelId(githubCopilotProvider().getModels(), 0);
let policyRequestCount = 0;
stubGitHubCopilotLoginFetch({
models: () =>
jsonResponse({
data: [{ id: "claude-sonnet-4.5", model_picker_enabled: true, policy: { state: "unconfigured" } }],
data: [{ id: modelId, model_picker_enabled: true, policy: { state: "unconfigured" } }],
}),
policy: () => {
policyRequestCount += 1;
@ -393,7 +412,8 @@ describe("GitHub Copilot OAuth device flow", () => {
it("continues policy updates after a transport failure", async () => {
vi.useFakeTimers();
const modelIds = ["gpt-4.1", "claude-sonnet-4.5"];
const providerModels = githubCopilotProvider().getModels();
const modelIds = [requireModelId(providerModels, 0), requireModelId(providerModels, 1)];
const policyModelIds: string[] = [];
stubGitHubCopilotLoginFetch({
models: () =>
@ -420,13 +440,16 @@ describe("GitHub Copilot OAuth device flow", () => {
it("stops policy updates and persists authentication when the retry delay exceeds the login budget", async () => {
vi.useFakeTimers();
const providerModels = githubCopilotProvider().getModels();
const firstModelId = requireModelId(providerModels, 0);
const secondModelId = requireModelId(providerModels, 1);
const policyModelIds: string[] = [];
stubGitHubCopilotLoginFetch({
models: () =>
jsonResponse({
data: [
{ id: "gpt-4.1", model_picker_enabled: true, policy: { state: "unconfigured" } },
{ id: "claude-sonnet-4.5", model_picker_enabled: true, policy: { state: "unconfigured" } },
{ id: firstModelId, model_picker_enabled: true, policy: { state: "unconfigured" } },
{ id: secondModelId, model_picker_enabled: true, policy: { state: "unconfigured" } },
],
}),
policy: (modelId) => {
@ -447,7 +470,7 @@ describe("GitHub Copilot OAuth device flow", () => {
await vi.advanceTimersByTimeAsync(1000);
const credential = await loginPromise;
expect(credential).toMatchObject({ type: "oauth", access: testCopilotAccessToken });
expect(policyModelIds).toEqual(["gpt-4.1"]);
expect(policyModelIds).toEqual([firstModelId]);
expect(await store.read("github-copilot")).toEqual(credential);
});

12
packages/ai/test/tool-call-id-normalization.test.ts
View file

@ -35,8 +35,8 @@ const echoTool: Tool<typeof echoToolSchema> = {
/**
* Test 1: Live cross-provider handoff
*
* 1. Use github-copilot gpt-5.2-codex to generate a tool call
* 2. Switch to openrouter openai/gpt-5.2-codex and complete
* 1. Use github-copilot gpt-5.5 to generate a tool call
* 2. Switch to openrouter openai/gpt-5.5 and complete
* 3. Switch to openai-codex gpt-5.5 and complete
*
* Both should succeed without "call_id too long" errors.
@ -45,8 +45,8 @@ describe("Tool Call ID Normalization - Live Handoff", () => {
it.skipIf(!copilotToken || !openrouterKey)(
"github-copilot -> openrouter should normalize pipe-separated IDs",
async () => {
const copilotModel = getModel("github-copilot", "gpt-5.2-codex");
const openrouterModel = getModel("openrouter", "openai/gpt-5.2-codex");
const copilotModel = getModel("github-copilot", "gpt-5.5");
const openrouterModel = getModel("openrouter", "openai/gpt-5.5");
// Step 1: Generate tool call with github-copilot
const userMessage: Message = {
@ -115,7 +115,7 @@ describe("Tool Call ID Normalization - Live Handoff", () => {
it.skipIf(!copilotToken || !codexToken)(
"github-copilot -> openai-codex should normalize pipe-separated IDs",
async () => {
const copilotModel = getModel("github-copilot", "gpt-5.2-codex");
const copilotModel = getModel("github-copilot", "gpt-5.5");
const codexModel = getModel("openai-codex", "gpt-5.5");
// Step 1: Generate tool call with github-copilot
@ -239,7 +239,7 @@ describe("Tool Call ID Normalization - Prefilled Context", () => {
it.skipIf(!openrouterKey)(
"openrouter should handle prefilled context with long pipe-separated IDs",
async () => {
const model = getModel("openrouter", "openai/gpt-5.2-codex");
const model = getModel("openrouter", "openai/gpt-5.5");
const messages = buildPrefilledMessages();
const response = await completeSimple(