mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 00:03:15 +01:00
* design: make images and read screenshots with Gemini when there is no OpenAI key (#990) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * design: test the live Gemini check; set fixture env keys without secret-shaped assignments Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes (setup keeps piped keys and names the provider export; Gemini iterate fallback tests) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * design: setup doc comment matches --key-stdin behavior Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: Gemini saves the final image of a thinking answer; iterate uploads only absolute-path PNGs under 15 MB; sessions store absolute output paths Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * changelog: fragment holds the entry text only Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * todos: Gemini design provider follow-ups Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: the egress fail-open list names design Gemini calls Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
179 lines
8.3 KiB
TypeScript
179 lines
8.3 KiB
TypeScript
/**
|
|
* The OpenAI models the design binary calls: one place, one override (#2807).
|
|
*
|
|
* Two call types, each with its own default:
|
|
* - "image": Responses API (`/v1/responses`) orchestrator driving the
|
|
* hosted `image_generation` tool — generate, variants, iterate, evolve.
|
|
* - "vision": Chat Completions (`/v1/chat/completions`) screenshot/mockup
|
|
* analysis — check, diff, memory, design-to-code, evolve's analysis step.
|
|
*
|
|
* `GSTACK_DESIGN_MODEL` (named like `GSTACK_CODEX_MODEL`) replaces both.
|
|
* `GSTACK_DESIGN_IMAGE_MODEL` replaces the image_generation tool's model; it
|
|
* must look like a gpt-image model name (IMAGE_TOOL_MODEL_PATTERN) and a bad
|
|
* value fails before any request, naming the variable.
|
|
*
|
|
* Defaults (checked against OpenAI's model and deprecation pages, 2026-10):
|
|
* gpt-5.5 lists Chat Completions, image input and the Responses
|
|
* image_generation tool, and has no announced shutdown. The tool's model is
|
|
* pinned to gpt-image-2: an unpinned tool falls back to gpt-image-1, which
|
|
* shuts down 2026-10-23. gpt-image-2 needs a gpt-5-class orchestrator (#1771:
|
|
* gpt-4o + gpt-image-2 is a 400), so an override to an older model fails
|
|
* before any request is sent, naming GSTACK_DESIGN_MODEL.
|
|
*
|
|
* Vision requests carry `max_completion_tokens`, never `max_tokens` (the
|
|
* gpt-5 family rejects it), and that limit covers reasoning tokens too: each
|
|
* call's visible-answer budget gets REASONING_HEADROOM_TOKENS on top, so a
|
|
* reasoning model cannot spend the whole limit thinking and return nothing.
|
|
* The limit is a cap, not a charge — only generated tokens are billed.
|
|
*
|
|
* `bun run design/scripts/live-model-check.ts` probes these defaults with a
|
|
* real key. It always tests the defaults (overrides are ignored), so an
|
|
* override cannot mask a retirement.
|
|
*/
|
|
|
|
export const DESIGN_MODEL_ENV = "GSTACK_DESIGN_MODEL";
|
|
|
|
export type DesignCall = "image" | "vision";
|
|
|
|
export const DEFAULT_DESIGN_MODELS: Readonly<Record<DesignCall, string>> = {
|
|
image: "gpt-5.5",
|
|
vision: "gpt-5.5",
|
|
};
|
|
|
|
/** The image model the hosted image_generation tool runs by default; never omitted (the API default is gpt-image-1). */
|
|
export const IMAGE_TOOL_MODEL = "gpt-image-2";
|
|
|
|
export const DESIGN_IMAGE_MODEL_ENV = "GSTACK_DESIGN_IMAGE_MODEL";
|
|
|
|
/** Conservative shape for an image tool model override: gpt-image-<version>[-variant]. */
|
|
export const IMAGE_TOOL_MODEL_PATTERN = /^gpt-image-[0-9][a-z0-9.-]{0,31}$/;
|
|
|
|
/**
|
|
* The image_generation tool's model: `GSTACK_DESIGN_IMAGE_MODEL` when set,
|
|
* else IMAGE_TOOL_MODEL. Throws, naming the variable, when the override does
|
|
* not match IMAGE_TOOL_MODEL_PATTERN.
|
|
*/
|
|
export function imageToolModel(env: Record<string, string | undefined> = process.env): string {
|
|
const override = env[DESIGN_IMAGE_MODEL_ENV]?.trim();
|
|
if (!override) return IMAGE_TOOL_MODEL;
|
|
if (!IMAGE_TOOL_MODEL_PATTERN.test(override)) {
|
|
throw new Error(`${DESIGN_IMAGE_MODEL_ENV}=${JSON.stringify(override.slice(0, 80))} is not a gpt-image model name ` +
|
|
`(expected something like ${IMAGE_TOOL_MODEL}); fix it or unset it to use ${IMAGE_TOOL_MODEL}`);
|
|
}
|
|
return override;
|
|
}
|
|
|
|
/**
|
|
* Gemini defaults (checked against https://ai.google.dev/gemini-api/docs/models,
|
|
* 2026-10-31): Nano Banana 2.1 is Google's current image generation and
|
|
* editing workhorse; Gemini 3.8 Flash reads images on the OpenAI-compatible
|
|
* endpoint. On Gemini, GSTACK_DESIGN_IMAGE_MODEL sets the image model and
|
|
* GSTACK_DESIGN_MODEL only the vision model (there is no orchestrator).
|
|
*/
|
|
export const GEMINI_DEFAULT_MODELS: Readonly<Record<DesignCall, string>> = {
|
|
image: "gemini-nano-banana-2.1",
|
|
vision: "gemini-3.8-flash",
|
|
};
|
|
|
|
export const GEMINI_MODEL_PATTERN = /^gemini-[a-z0-9][a-z0-9.-]{0,40}$/;
|
|
|
|
/**
|
|
* The Gemini model for one call type. An override that is not a gemini-…
|
|
* name (an OpenAI model left in the environment, say) fails before any
|
|
* request, naming the variable.
|
|
*/
|
|
export function geminiModel(call: DesignCall, env: Record<string, string | undefined> = process.env): string {
|
|
const variable = call === "image" ? DESIGN_IMAGE_MODEL_ENV : DESIGN_MODEL_ENV;
|
|
const override = env[variable]?.trim();
|
|
if (!override) return GEMINI_DEFAULT_MODELS[call];
|
|
if (!GEMINI_MODEL_PATTERN.test(override)) {
|
|
throw new Error(`${variable}=${JSON.stringify(override.slice(0, 80))} is not a Gemini model name, and the design provider is gemini.\n` +
|
|
`Fix: unset it or set a gemini-… model (on Gemini, ${DESIGN_IMAGE_MODEL_ENV} sets the image model and ${DESIGN_MODEL_ENV} the vision model). See docs/troubleshooting.md#design-image-model-invalid`);
|
|
}
|
|
return override;
|
|
}
|
|
|
|
/** The OpenAI image sizes the design commands use, as Gemini aspect ratios (Gemini picks the pixel size; --quality has no Gemini meaning). */
|
|
export const GEMINI_ASPECT_RATIOS: Readonly<Record<string, string>> = {
|
|
"1536x1024": "3:2",
|
|
"1024x1024": "1:1",
|
|
"1024x1536": "2:3",
|
|
};
|
|
|
|
export function geminiAspectRatio(size: string): string {
|
|
const ratio = GEMINI_ASPECT_RATIOS[size];
|
|
if (!ratio) throw new Error(`--size ${size} has no Gemini aspect ratio; use one of ${Object.keys(GEMINI_ASPECT_RATIOS).join(", ")}.`);
|
|
return ratio;
|
|
}
|
|
|
|
/** Reasoning allowance added to every vision call's visible-answer budget. */
|
|
export const REASONING_HEADROOM_TOKENS = 8192;
|
|
|
|
/** The model for one call type: `GSTACK_DESIGN_MODEL` when set, else that call type's default. */
|
|
export function designModel(call: DesignCall, env: Record<string, string | undefined> = process.env): string {
|
|
return env[DESIGN_MODEL_ENV]?.trim() || DEFAULT_DESIGN_MODELS[call];
|
|
}
|
|
|
|
/** Why `orchestrator` cannot drive the image_generation tool model, or null when it can. */
|
|
export function imagePairingProblem(orchestrator: string, toolModel = IMAGE_TOOL_MODEL): string | null {
|
|
if (!/^(?:gpt-4|gpt-3|chatgpt-4o)/i.test(orchestrator)) return null;
|
|
return `${orchestrator} cannot drive the ${toolModel} image_generation tool (OpenAI returns 400 for that pairing); ` +
|
|
`set ${DESIGN_MODEL_ENV} to a gpt-5-class model such as ${DEFAULT_DESIGN_MODELS.image}, or unset it`;
|
|
}
|
|
|
|
/**
|
|
* Responses API body for an image call: the image orchestrator plus the
|
|
* image_generation tool with its model pinned. Throws (before any request)
|
|
* when the orchestrator cannot drive that tool model.
|
|
*/
|
|
export function imageRequestBody(
|
|
input: string,
|
|
tool: { size: string; quality: string },
|
|
extra: Record<string, unknown> = {},
|
|
): string {
|
|
const model = designModel("image");
|
|
const toolModel = imageToolModel();
|
|
const problem = imagePairingProblem(model, toolModel);
|
|
if (problem) throw new Error(problem);
|
|
return JSON.stringify({
|
|
model,
|
|
input,
|
|
...extra,
|
|
tools: [{ type: "image_generation", model: toolModel, size: tool.size, quality: tool.quality }],
|
|
});
|
|
}
|
|
|
|
/** Chat Completions body for a vision call; `visibleTokens` is the answer's budget, reasoning headroom is added. */
|
|
export function visionRequestBody(
|
|
messages: unknown[],
|
|
visibleTokens: number,
|
|
extra: Record<string, unknown> = {},
|
|
): string {
|
|
return JSON.stringify({
|
|
model: designModel("vision"),
|
|
messages,
|
|
max_completion_tokens: visibleTokens + REASONING_HEADROOM_TOKENS,
|
|
...extra,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Suffix for an OpenAI error message when the failure is about the model or a
|
|
* parameter the model rejects: names the model sent and `GSTACK_DESIGN_MODEL`,
|
|
* so a rejected override (or a retired default) says how to fix it. Empty for
|
|
* every other failure.
|
|
*/
|
|
export function modelRejectionHint(status: number, body: string, call: DesignCall): string {
|
|
if (![400, 403, 404].includes(status)) return "";
|
|
if (!/\bmodel\b|unsupported (?:parameter|value)|not supported/i.test(body)) return "";
|
|
const model = designModel(call);
|
|
const source = process.env[DESIGN_MODEL_ENV]?.trim() ? `${DESIGN_MODEL_ENV}=${model}` : `the default model ${model}`;
|
|
if (call !== "image") {
|
|
return ` — OpenAI rejected ${source} for this vision (Chat Completions) call; set ${DESIGN_MODEL_ENV} to a model your account can use for it`;
|
|
}
|
|
const toolOverride = process.env[DESIGN_IMAGE_MODEL_ENV]?.trim();
|
|
const tool = toolOverride ? `${DESIGN_IMAGE_MODEL_ENV}=${toolOverride}` : `the default tool model ${IMAGE_TOOL_MODEL}`;
|
|
return ` — OpenAI rejected ${source} or ${tool} for this image-generation (Responses API) call; ` +
|
|
`set ${DESIGN_MODEL_ENV} or ${DESIGN_IMAGE_MODEL_ENV} to models your account can use for it`;
|
|
}
|