diff --git a/design/scripts/live-model-check.ts b/design/scripts/live-model-check.ts index e887afc06..111a18587 100644 --- a/design/scripts/live-model-check.ts +++ b/design/scripts/live-model-check.ts @@ -1,31 +1,39 @@ /** - * Opt-in live check of the design binary's OpenAI defaults (#2807). Not part - * of any test lane: it spends real tokens (one low-quality 1024x1024 image, - * one short vision answer). + * Live check of the design binary's OpenAI defaults (#2807, E5). It spends + * real tokens (one low-quality 1024x1024 image, one short vision answer). * * OPENAI_API_KEY=... bun run design/scripts/live-model-check.ts * * Sends exactly the request bodies the design commands send (built by - * design/src/models.ts), honoring GSTACK_DESIGN_MODEL when set. Prints the - * models requested and returned, HTTP status, token usage and PASS/FAIL per - * call — never the key, and any key-shaped text in an error body is redacted. - * Exit 0 both pass, 1 any fail, 2 OPENAI_API_KEY unset. + * design/src/models.ts) with the DEFAULT models: GSTACK_DESIGN_MODEL and + * GSTACK_DESIGN_IMAGE_MODEL are ignored (and named when set), so an override + * cannot mask a retirement of a default. Prints the models requested and + * returned, HTTP status, token usage and PASS/FAIL per call — never the key, + * and any key-shaped text in an error body is redacted. + * Exit 0 both pass, 1 any fail, 2 OPENAI_API_KEY unset: a skip that checked + * nothing, which a caller must record as skipped with no coverage credit. */ import { + DESIGN_IMAGE_MODEL_ENV, DESIGN_MODEL_ENV, - IMAGE_TOOL_MODEL, designModel, imageRequestBody, + imageToolModel, visionRequestBody, } from "../src/models"; import { receiptedFetch } from "../src/receipted-fetch"; const TINY_PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkAAIAAAoAAv/lxKUAAAAASUVORK5CYII="; +const ignored = [DESIGN_MODEL_ENV, DESIGN_IMAGE_MODEL_ENV].filter((name) => process.env[name]?.trim()); +for (const name of ignored) delete process.env[name]; +console.log(`testing defaults: image=${designModel("image")} tool=${imageToolModel()} vision=${designModel("vision")}` + + (ignored.length ? ` (ignored: ${ignored.join(", ")})` : "")); + const key = process.env.OPENAI_API_KEY?.trim(); if (!key) { - console.log("SKIP: OPENAI_API_KEY is not set"); + console.log("SKIP: OPENAI_API_KEY is not set; the design model defaults were not checked (no coverage)"); process.exit(2); } @@ -56,7 +64,6 @@ async function post(payloadClass: string, url: string, body: string, timeoutMs: const report = (fields: Record) => console.log(Object.entries(fields).map(([k, v]) => `${k}=${typeof v === "string" ? v : JSON.stringify(v)}`).join(" ")); -console.log(`${DESIGN_MODEL_ENV}=${process.env[DESIGN_MODEL_ENV]?.trim() || "(unset, defaults)"}`); let failed = false; let image = TINY_PNG; @@ -78,7 +85,7 @@ let image = TINY_PNG; report({ call: "image", result: pass ? "PASS" : "FAIL", status: r.status, ms: r.ms, requested_model: designModel("image"), returned_model: r.json?.model ?? null, - tool_model: IMAGE_TOOL_MODEL, image_bytes: pass ? Buffer.from(item.result, "base64").length : 0, + tool_model: imageToolModel(), image_bytes: pass ? Buffer.from(item.result, "base64").length : 0, ...(pass ? {} : { error: redact(r.json?.error?.message ?? r.text) }), }); failed ||= !pass; diff --git a/design/src/models.ts b/design/src/models.ts index 081709eb8..5fed94daf 100644 --- a/design/src/models.ts +++ b/design/src/models.ts @@ -8,6 +8,9 @@ * analysis — check, diff, memory, design-to-code, evolve's analysis step. * * `GSTACK_DESIGN_MODEL` (named like `GSTACK_CODEX_MODEL`) replaces both. + * `GSTACK_DESIGN_IMAGE_MODEL` replaces the image_generation tool's model; it + * must look like a gpt-image model name (IMAGE_TOOL_MODEL_PATTERN) and a bad + * value fails before any request, naming the variable. * * Defaults (checked against OpenAI's model and deprecation pages, 2026-10): * gpt-5.5 lists Chat Completions, image input and the Responses @@ -24,7 +27,8 @@ * The limit is a cap, not a charge — only generated tokens are billed. * * `bun run design/scripts/live-model-check.ts` probes these defaults with a - * real key (opt-in, not in any test lane). + * real key. It always tests the defaults (overrides are ignored), so an + * override cannot mask a retirement. */ export const DESIGN_MODEL_ENV = "GSTACK_DESIGN_MODEL"; @@ -36,9 +40,29 @@ export const DEFAULT_DESIGN_MODELS: Readonly> = { vision: "gpt-5.5", }; -/** The image model the hosted image_generation tool runs; never omitted (the API default is gpt-image-1). */ +/** The image model the hosted image_generation tool runs by default; never omitted (the API default is gpt-image-1). */ export const IMAGE_TOOL_MODEL = "gpt-image-2"; +export const DESIGN_IMAGE_MODEL_ENV = "GSTACK_DESIGN_IMAGE_MODEL"; + +/** Conservative shape for an image tool model override: gpt-image-[-variant]. */ +export const IMAGE_TOOL_MODEL_PATTERN = /^gpt-image-[0-9][a-z0-9.-]{0,31}$/; + +/** + * The image_generation tool's model: `GSTACK_DESIGN_IMAGE_MODEL` when set, + * else IMAGE_TOOL_MODEL. Throws, naming the variable, when the override does + * not match IMAGE_TOOL_MODEL_PATTERN. + */ +export function imageToolModel(env: Record = process.env): string { + const override = env[DESIGN_IMAGE_MODEL_ENV]?.trim(); + if (!override) return IMAGE_TOOL_MODEL; + if (!IMAGE_TOOL_MODEL_PATTERN.test(override)) { + throw new Error(`${DESIGN_IMAGE_MODEL_ENV}=${JSON.stringify(override.slice(0, 80))} is not a gpt-image model name ` + + `(expected something like ${IMAGE_TOOL_MODEL}); fix it or unset it to use ${IMAGE_TOOL_MODEL}`); + } + return override; +} + /** Reasoning allowance added to every vision call's visible-answer budget. */ export const REASONING_HEADROOM_TOKENS = 8192; @@ -47,10 +71,10 @@ export function designModel(call: DesignCall, env: Record = {}, ): string { const model = designModel("image"); - const problem = imagePairingProblem(model); + const toolModel = imageToolModel(); + const problem = imagePairingProblem(model, toolModel); if (problem) throw new Error(problem); return JSON.stringify({ model, input, ...extra, - tools: [{ type: "image_generation", model: IMAGE_TOOL_MODEL, size: tool.size, quality: tool.quality }], + tools: [{ type: "image_generation", model: toolModel, size: tool.size, quality: tool.quality }], }); } @@ -100,8 +125,11 @@ export function modelRejectionHint(status: number, body: string, call: DesignCal if (!/\bmodel\b|unsupported (?:parameter|value)|not supported/i.test(body)) return ""; const model = designModel(call); const source = process.env[DESIGN_MODEL_ENV]?.trim() ? `${DESIGN_MODEL_ENV}=${model}` : `the default model ${model}`; - const what = call === "image" - ? `image-generation (Responses API, tool model ${IMAGE_TOOL_MODEL})` - : "vision (Chat Completions)"; - return ` — OpenAI rejected ${source} for this ${what} call; set ${DESIGN_MODEL_ENV} to a model your account can use for it`; + if (call !== "image") { + return ` — OpenAI rejected ${source} for this vision (Chat Completions) call; set ${DESIGN_MODEL_ENV} to a model your account can use for it`; + } + const toolOverride = process.env[DESIGN_IMAGE_MODEL_ENV]?.trim(); + const tool = toolOverride ? `${DESIGN_IMAGE_MODEL_ENV}=${toolOverride}` : `the default tool model ${IMAGE_TOOL_MODEL}`; + return ` — OpenAI rejected ${source} or ${tool} for this image-generation (Responses API) call; ` + + `set ${DESIGN_MODEL_ENV} or ${DESIGN_IMAGE_MODEL_ENV} to models your account can use for it`; } diff --git a/design/test/image-gen-pairing.test.ts b/design/test/image-gen-pairing.test.ts index 8e72ded15..dba74642a 100644 --- a/design/test/image-gen-pairing.test.ts +++ b/design/test/image-gen-pairing.test.ts @@ -1,13 +1,17 @@ import { afterEach, describe, expect, test } from "bun:test"; +import { spawnSync } from "child_process"; import fs from "fs"; +import os from "os"; import path from "path"; import { DEFAULT_DESIGN_MODELS, + DESIGN_IMAGE_MODEL_ENV, DESIGN_MODEL_ENV, IMAGE_TOOL_MODEL, imagePairingProblem, imageRequestBody, + imageToolModel, } from "../src/models"; // Tripwire for the image_generation orchestrator/tool pairing. @@ -33,9 +37,12 @@ const sources = fs .map((f) => ({ rel: `src/${f}`, body: fs.readFileSync(path.join(DESIGN_SRC, f), "utf-8") })); const savedOverride = process.env[DESIGN_MODEL_ENV]; +const savedImageOverride = process.env[DESIGN_IMAGE_MODEL_ENV]; afterEach(() => { if (savedOverride === undefined) delete process.env[DESIGN_MODEL_ENV]; else process.env[DESIGN_MODEL_ENV] = savedOverride; + if (savedImageOverride === undefined) delete process.env[DESIGN_IMAGE_MODEL_ENV]; + else process.env[DESIGN_IMAGE_MODEL_ENV] = savedImageOverride; }); describe("design image-generation tool/orchestrator pairing (#1771, #2807)", () => { @@ -71,3 +78,72 @@ describe("design image-generation tool/orchestrator pairing (#1771, #2807)", () }); } }); + +// E5: users switch the tool model without editing code; a bad value fails +// before any request, naming the variable; the live check always tests the +// defaults, and a missing key is a skip, never a pass. +describe("GSTACK_DESIGN_IMAGE_MODEL override (E5)", () => { + const body = () => JSON.parse(imageRequestBody("x", { size: "1024x1024", quality: "low" })); + + test("a gpt-image model name replaces the tool model; blank means the default", () => { + delete process.env[DESIGN_MODEL_ENV]; + for (const model of ["gpt-image-1.5", "gpt-image-2-mini", "gpt-image-3"]) { + process.env[DESIGN_IMAGE_MODEL_ENV] = ` ${model} `; + expect(body().tools[0].model).toBe(model); + } + process.env[DESIGN_IMAGE_MODEL_ENV] = " "; + expect(body().tools[0].model).toBe(IMAGE_TOOL_MODEL); + expect(imageToolModel({})).toBe(IMAGE_TOOL_MODEL); + }); + + test("anything else is refused before a request, naming the variable", () => { + for (const bad of ["dall-e-3", "gpt-5.5", "GPT-IMAGE-2", "gpt-image-2 --x", 'gpt-image-2"}', "gpt-image-", `gpt-image-2${"a".repeat(40)}`]) { + process.env[DESIGN_IMAGE_MODEL_ENV] = bad; + expect(() => body()).toThrow(`${DESIGN_IMAGE_MODEL_ENV}=`); + expect(() => imageToolModel()).toThrow(`unset it to use ${IMAGE_TOOL_MODEL}`); + } + }); + + test("the pairing refusal names the tool model actually requested", () => { + process.env[DESIGN_MODEL_ENV] = "gpt-4o"; + process.env[DESIGN_IMAGE_MODEL_ENV] = "gpt-image-1.5"; + expect(() => body()).toThrow("cannot drive the gpt-image-1.5 image_generation tool"); + }); +}); + +describe("live-model-check always tests the defaults (E5)", () => { + const script = path.join(import.meta.dir, "..", "scripts", "live-model-check.ts"); + const run = (env: Record) => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "design-live-check-")); + try { + return spawnSync(process.execPath, ["run", script], { + encoding: "utf8", timeout: 60_000, + env: { PATH: process.env.PATH ?? "/usr/bin:/bin", HOME: home, GSTACK_HOME: home, ...env }, + }); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }; + const overrides = { [DESIGN_MODEL_ENV]: "gpt-4.1", [DESIGN_IMAGE_MODEL_ENV]: "gpt-image-1" }; + + test("no key: exit 2 with a no-coverage SKIP line, after naming the ignored overrides", () => { + const r = run(overrides); + expect(r.status).toBe(2); + expect(r.stdout.trim().split("\n")).toEqual([ + `testing defaults: image=${DEFAULT_DESIGN_MODELS.image} tool=${IMAGE_TOOL_MODEL} vision=${DEFAULT_DESIGN_MODELS.vision} (ignored: ${DESIGN_MODEL_ENV}, ${DESIGN_IMAGE_MODEL_ENV})`, + "SKIP: OPENAI_API_KEY is not set; the design model defaults were not checked (no coverage)", + ]); + }); + + test("with a key, the requests carry the defaults even when overrides are set", () => { + // An unreachable proxy fails both calls locally: no request leaves the machine. + const r = run({ ...overrides, OPENAI_API_KEY: "sk-test-not-a-real-key", HTTPS_PROXY: "http://127.0.0.1:9", https_proxy: "http://127.0.0.1:9" }); + expect(r.status).toBe(1); + const image = r.stdout.split("\n").find((l) => l.startsWith("call=image")) ?? ""; + const vision = r.stdout.split("\n").find((l) => l.startsWith("call=vision")) ?? ""; + expect(image).toContain(`requested_model=${DEFAULT_DESIGN_MODELS.image}`); + expect(image).toContain(`tool_model=${IMAGE_TOOL_MODEL}`); + expect(vision).toContain(`requested_model=${DEFAULT_DESIGN_MODELS.vision}`); + expect(r.stdout).not.toContain("sk-test-not-a-real-key"); + }); +});