E5: GSTACK_DESIGN_IMAGE_MODEL override; the live model check always tests the defaults

The image_generation tool model gains an env override validated against a
conservative gpt-image-<version> pattern; a bad value fails before any
request and the error names the variable. Pairing and rejection hints name
the tool model actually requested. design/scripts/live-model-check.ts now
ignores GSTACK_DESIGN_MODEL and GSTACK_DESIGN_IMAGE_MODEL (and says so), so
an override cannot mask a retirement of a default, and its exit 2 (no key)
says nothing was checked.

design/scripts/live-model-check.ts

@@ -1,31 +1,39 @@
 /**
- * Opt-in live check of the design binary's OpenAI defaults (#2807). Not part
- * of any test lane: it spends real tokens (one low-quality 1024x1024 image,
- * one short vision answer).
+ * Live check of the design binary's OpenAI defaults (#2807, E5). It spends
+ * real tokens (one low-quality 1024x1024 image, one short vision answer).
  *
  *   OPENAI_API_KEY=... bun run design/scripts/live-model-check.ts
  *
  * Sends exactly the request bodies the design commands send (built by
- * design/src/models.ts), honoring GSTACK_DESIGN_MODEL when set. Prints the
- * models requested and returned, HTTP status, token usage and PASS/FAIL per
- * call — never the key, and any key-shaped text in an error body is redacted.
- * Exit 0 both pass, 1 any fail, 2 OPENAI_API_KEY unset.
+ * design/src/models.ts) with the DEFAULT models: GSTACK_DESIGN_MODEL and
+ * GSTACK_DESIGN_IMAGE_MODEL are ignored (and named when set), so an override
+ * cannot mask a retirement of a default. Prints the models requested and
+ * returned, HTTP status, token usage and PASS/FAIL per call — never the key,
+ * and any key-shaped text in an error body is redacted.
+ * Exit 0 both pass, 1 any fail, 2 OPENAI_API_KEY unset: a skip that checked
+ * nothing, which a caller must record as skipped with no coverage credit.
  */
 
 import {
+  DESIGN_IMAGE_MODEL_ENV,
   DESIGN_MODEL_ENV,
-  IMAGE_TOOL_MODEL,
   designModel,
   imageRequestBody,
+  imageToolModel,
   visionRequestBody,
 } from "../src/models";
 import { receiptedFetch } from "../src/receipted-fetch";
 
 const TINY_PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkAAIAAAoAAv/lxKUAAAAASUVORK5CYII=";
 
+const ignored = [DESIGN_MODEL_ENV, DESIGN_IMAGE_MODEL_ENV].filter((name) => process.env[name]?.trim());
+for (const name of ignored) delete process.env[name];
+console.log(`testing defaults: image=${designModel("image")} tool=${imageToolModel()} vision=${designModel("vision")}` +
+  (ignored.length ? ` (ignored: ${ignored.join(", ")})` : ""));
+
 const key = process.env.OPENAI_API_KEY?.trim();
 if (!key) {
-  console.log("SKIP: OPENAI_API_KEY is not set");
+  console.log("SKIP: OPENAI_API_KEY is not set; the design model defaults were not checked (no coverage)");
   process.exit(2);
 }
 
@@ -56,7 +64,6 @@ async function post(payloadClass: string, url: string, body: string, timeoutMs:
 const report = (fields: Record<string, unknown>) =>
   console.log(Object.entries(fields).map(([k, v]) => `${k}=${typeof v === "string" ? v : JSON.stringify(v)}`).join(" "));
 
-console.log(`${DESIGN_MODEL_ENV}=${process.env[DESIGN_MODEL_ENV]?.trim() || "(unset, defaults)"}`);
 let failed = false;
 let image = TINY_PNG;
 
@@ -78,7 +85,7 @@ let image = TINY_PNG;
     report({
       call: "image", result: pass ? "PASS" : "FAIL", status: r.status, ms: r.ms,
       requested_model: designModel("image"), returned_model: r.json?.model ?? null,
-      tool_model: IMAGE_TOOL_MODEL, image_bytes: pass ? Buffer.from(item.result, "base64").length : 0,
+      tool_model: imageToolModel(), image_bytes: pass ? Buffer.from(item.result, "base64").length : 0,
       ...(pass ? {} : { error: redact(r.json?.error?.message ?? r.text) }),
     });
     failed ||= !pass;

design/src/models.ts

@@ -8,6 +8,9 @@
  *     analysis — check, diff, memory, design-to-code, evolve's analysis step.
  *
  * `GSTACK_DESIGN_MODEL` (named like `GSTACK_CODEX_MODEL`) replaces both.
+ * `GSTACK_DESIGN_IMAGE_MODEL` replaces the image_generation tool's model; it
+ * must look like a gpt-image model name (IMAGE_TOOL_MODEL_PATTERN) and a bad
+ * value fails before any request, naming the variable.
  *
  * Defaults (checked against OpenAI's model and deprecation pages, 2026-10):
  * gpt-5.5 lists Chat Completions, image input and the Responses
@@ -24,7 +27,8 @@
  * The limit is a cap, not a charge — only generated tokens are billed.
  *
  * `bun run design/scripts/live-model-check.ts` probes these defaults with a
- * real key (opt-in, not in any test lane).
+ * real key. It always tests the defaults (overrides are ignored), so an
+ * override cannot mask a retirement.
  */
 
 export const DESIGN_MODEL_ENV = "GSTACK_DESIGN_MODEL";
@@ -36,9 +40,29 @@ export const DEFAULT_DESIGN_MODELS: Readonly<Record<DesignCall, string>> = {
   vision: "gpt-5.5",
 };
 
-/** The image model the hosted image_generation tool runs; never omitted (the API default is gpt-image-1). */
+/** The image model the hosted image_generation tool runs by default; never omitted (the API default is gpt-image-1). */
 export const IMAGE_TOOL_MODEL = "gpt-image-2";
 
+export const DESIGN_IMAGE_MODEL_ENV = "GSTACK_DESIGN_IMAGE_MODEL";
+
+/** Conservative shape for an image tool model override: gpt-image-<version>[-variant]. */
+export const IMAGE_TOOL_MODEL_PATTERN = /^gpt-image-[0-9][a-z0-9.-]{0,31}$/;
+
+/**
+ * The image_generation tool's model: `GSTACK_DESIGN_IMAGE_MODEL` when set,
+ * else IMAGE_TOOL_MODEL. Throws, naming the variable, when the override does
+ * not match IMAGE_TOOL_MODEL_PATTERN.
+ */
+export function imageToolModel(env: Record<string, string | undefined> = process.env): string {
+  const override = env[DESIGN_IMAGE_MODEL_ENV]?.trim();
+  if (!override) return IMAGE_TOOL_MODEL;
+  if (!IMAGE_TOOL_MODEL_PATTERN.test(override)) {
+    throw new Error(`${DESIGN_IMAGE_MODEL_ENV}=${JSON.stringify(override.slice(0, 80))} is not a gpt-image model name ` +
+      `(expected something like ${IMAGE_TOOL_MODEL}); fix it or unset it to use ${IMAGE_TOOL_MODEL}`);
+  }
+  return override;
+}
+
 /** Reasoning allowance added to every vision call's visible-answer budget. */
 export const REASONING_HEADROOM_TOKENS = 8192;
 
@@ -47,10 +71,10 @@ export function designModel(call: DesignCall, env: Record<string, string | undef
   return env[DESIGN_MODEL_ENV]?.trim() || DEFAULT_DESIGN_MODELS[call];
 }
 
-/** Why `orchestrator` cannot drive the IMAGE_TOOL_MODEL tool, or null when it can. */
-export function imagePairingProblem(orchestrator: string): string | null {
+/** Why `orchestrator` cannot drive the image_generation tool model, or null when it can. */
+export function imagePairingProblem(orchestrator: string, toolModel = IMAGE_TOOL_MODEL): string | null {
   if (!/^(?:gpt-4|gpt-3|chatgpt-4o)/i.test(orchestrator)) return null;
-  return `${orchestrator} cannot drive the ${IMAGE_TOOL_MODEL} image_generation tool (OpenAI returns 400 for that pairing); ` +
+  return `${orchestrator} cannot drive the ${toolModel} image_generation tool (OpenAI returns 400 for that pairing); ` +
     `set ${DESIGN_MODEL_ENV} to a gpt-5-class model such as ${DEFAULT_DESIGN_MODELS.image}, or unset it`;
 }
 
@@ -65,13 +89,14 @@ export function imageRequestBody(
   extra: Record<string, unknown> = {},
 ): string {
   const model = designModel("image");
-  const problem = imagePairingProblem(model);
+  const toolModel = imageToolModel();
+  const problem = imagePairingProblem(model, toolModel);
   if (problem) throw new Error(problem);
   return JSON.stringify({
     model,
     input,
     ...extra,
-    tools: [{ type: "image_generation", model: IMAGE_TOOL_MODEL, size: tool.size, quality: tool.quality }],
+    tools: [{ type: "image_generation", model: toolModel, size: tool.size, quality: tool.quality }],
   });
 }
 
@@ -100,8 +125,11 @@ export function modelRejectionHint(status: number, body: string, call: DesignCal
   if (!/\bmodel\b|unsupported (?:parameter|value)|not supported/i.test(body)) return "";
   const model = designModel(call);
   const source = process.env[DESIGN_MODEL_ENV]?.trim() ? `${DESIGN_MODEL_ENV}=${model}` : `the default model ${model}`;
-  const what = call === "image"
-    ? `image-generation (Responses API, tool model ${IMAGE_TOOL_MODEL})`
-    : "vision (Chat Completions)";
-  return ` — OpenAI rejected ${source} for this ${what} call; set ${DESIGN_MODEL_ENV} to a model your account can use for it`;
+  if (call !== "image") {
+    return ` — OpenAI rejected ${source} for this vision (Chat Completions) call; set ${DESIGN_MODEL_ENV} to a model your account can use for it`;
+  }
+  const toolOverride = process.env[DESIGN_IMAGE_MODEL_ENV]?.trim();
+  const tool = toolOverride ? `${DESIGN_IMAGE_MODEL_ENV}=${toolOverride}` : `the default tool model ${IMAGE_TOOL_MODEL}`;
+  return ` — OpenAI rejected ${source} or ${tool} for this image-generation (Responses API) call; ` +
+    `set ${DESIGN_MODEL_ENV} or ${DESIGN_IMAGE_MODEL_ENV} to models your account can use for it`;
 }

design/test/image-gen-pairing.test.ts

@@ -1,13 +1,17 @@
 import { afterEach, describe, expect, test } from "bun:test";
+import { spawnSync } from "child_process";
 import fs from "fs";
+import os from "os";
 import path from "path";
 
 import {
   DEFAULT_DESIGN_MODELS,
+  DESIGN_IMAGE_MODEL_ENV,
   DESIGN_MODEL_ENV,
   IMAGE_TOOL_MODEL,
   imagePairingProblem,
   imageRequestBody,
+  imageToolModel,
 } from "../src/models";
 
 // Tripwire for the image_generation orchestrator/tool pairing.
@@ -33,9 +37,12 @@ const sources = fs
   .map((f) => ({ rel: `src/${f}`, body: fs.readFileSync(path.join(DESIGN_SRC, f), "utf-8") }));
 
 const savedOverride = process.env[DESIGN_MODEL_ENV];
+const savedImageOverride = process.env[DESIGN_IMAGE_MODEL_ENV];
 afterEach(() => {
   if (savedOverride === undefined) delete process.env[DESIGN_MODEL_ENV];
   else process.env[DESIGN_MODEL_ENV] = savedOverride;
+  if (savedImageOverride === undefined) delete process.env[DESIGN_IMAGE_MODEL_ENV];
+  else process.env[DESIGN_IMAGE_MODEL_ENV] = savedImageOverride;
 });
 
 describe("design image-generation tool/orchestrator pairing (#1771, #2807)", () => {
@@ -71,3 +78,72 @@ describe("design image-generation tool/orchestrator pairing (#1771, #2807)", ()
     });
   }
 });
+
+// E5: users switch the tool model without editing code; a bad value fails
+// before any request, naming the variable; the live check always tests the
+// defaults, and a missing key is a skip, never a pass.
+describe("GSTACK_DESIGN_IMAGE_MODEL override (E5)", () => {
+  const body = () => JSON.parse(imageRequestBody("x", { size: "1024x1024", quality: "low" }));
+
+  test("a gpt-image model name replaces the tool model; blank means the default", () => {
+    delete process.env[DESIGN_MODEL_ENV];
+    for (const model of ["gpt-image-1.5", "gpt-image-2-mini", "gpt-image-3"]) {
+      process.env[DESIGN_IMAGE_MODEL_ENV] = ` ${model} `;
+      expect(body().tools[0].model).toBe(model);
+    }
+    process.env[DESIGN_IMAGE_MODEL_ENV] = "   ";
+    expect(body().tools[0].model).toBe(IMAGE_TOOL_MODEL);
+    expect(imageToolModel({})).toBe(IMAGE_TOOL_MODEL);
+  });
+
+  test("anything else is refused before a request, naming the variable", () => {
+    for (const bad of ["dall-e-3", "gpt-5.5", "GPT-IMAGE-2", "gpt-image-2 --x", 'gpt-image-2"}', "gpt-image-", `gpt-image-2${"a".repeat(40)}`]) {
+      process.env[DESIGN_IMAGE_MODEL_ENV] = bad;
+      expect(() => body()).toThrow(`${DESIGN_IMAGE_MODEL_ENV}=`);
+      expect(() => imageToolModel()).toThrow(`unset it to use ${IMAGE_TOOL_MODEL}`);
+    }
+  });
+
+  test("the pairing refusal names the tool model actually requested", () => {
+    process.env[DESIGN_MODEL_ENV] = "gpt-4o";
+    process.env[DESIGN_IMAGE_MODEL_ENV] = "gpt-image-1.5";
+    expect(() => body()).toThrow("cannot drive the gpt-image-1.5 image_generation tool");
+  });
+});
+
+describe("live-model-check always tests the defaults (E5)", () => {
+  const script = path.join(import.meta.dir, "..", "scripts", "live-model-check.ts");
+  const run = (env: Record<string, string>) => {
+    const home = fs.mkdtempSync(path.join(os.tmpdir(), "design-live-check-"));
+    try {
+      return spawnSync(process.execPath, ["run", script], {
+        encoding: "utf8", timeout: 60_000,
+        env: { PATH: process.env.PATH ?? "/usr/bin:/bin", HOME: home, GSTACK_HOME: home, ...env },
+      });
+    } finally {
+      fs.rmSync(home, { recursive: true, force: true });
+    }
+  };
+  const overrides = { [DESIGN_MODEL_ENV]: "gpt-4.1", [DESIGN_IMAGE_MODEL_ENV]: "gpt-image-1" };
+
+  test("no key: exit 2 with a no-coverage SKIP line, after naming the ignored overrides", () => {
+    const r = run(overrides);
+    expect(r.status).toBe(2);
+    expect(r.stdout.trim().split("\n")).toEqual([
+      `testing defaults: image=${DEFAULT_DESIGN_MODELS.image} tool=${IMAGE_TOOL_MODEL} vision=${DEFAULT_DESIGN_MODELS.vision} (ignored: ${DESIGN_MODEL_ENV}, ${DESIGN_IMAGE_MODEL_ENV})`,
+      "SKIP: OPENAI_API_KEY is not set; the design model defaults were not checked (no coverage)",
+    ]);
+  });
+
+  test("with a key, the requests carry the defaults even when overrides are set", () => {
+    // An unreachable proxy fails both calls locally: no request leaves the machine.
+    const r = run({ ...overrides, OPENAI_API_KEY: "sk-test-not-a-real-key", HTTPS_PROXY: "http://127.0.0.1:9", https_proxy: "http://127.0.0.1:9" });
+    expect(r.status).toBe(1);
+    const image = r.stdout.split("\n").find((l) => l.startsWith("call=image")) ?? "";
+    const vision = r.stdout.split("\n").find((l) => l.startsWith("call=vision")) ?? "";
+    expect(image).toContain(`requested_model=${DEFAULT_DESIGN_MODELS.image}`);
+    expect(image).toContain(`tool_model=${IMAGE_TOOL_MODEL}`);
+    expect(vision).toContain(`requested_model=${DEFAULT_DESIGN_MODELS.vision}`);
+    expect(r.stdout).not.toContain("sk-test-not-a-real-key");
+  });
+});