Files
gstack/design/src/provider.ts
T
Garry Tanandcapy-ai[bot] 71acd83aa6 v1.92.11.0 feat: design mockups with Gemini when there's no OpenAI key (#990) (#3768)
* design: make images and read screenshots with Gemini when there is no OpenAI key (#990)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* design: test the live Gemini check; set fixture env keys without secret-shaped assignments

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: pre-landing review fixes (setup keeps piped keys and names the provider export; Gemini iterate fallback tests)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* design: setup doc comment matches --key-stdin behavior

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: Gemini saves the final image of a thinking answer; iterate uploads only absolute-path PNGs under 15 MB; sessions store absolute output paths

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* changelog: fragment holds the entry text only

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* todos: Gemini design provider follow-ups

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* docs: the egress fail-open list names design Gemini calls

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
2026-11-01 05:17:06 +00:00

205 lines
10 KiB
TypeScript

/**
* The design binary's provider seam: which image service a run talks to
* (OpenAI or Gemini, picked by auth.ts resolveProvider) and how each call's
* request is built and its response read. Call sites keep their own fetch,
* timeouts and retry loops; this module only shapes requests, responses and
* errors, so OpenAI requests stay byte-identical to what they were.
*
* Gemini image calls use the native generateContent API
* (POST /v1beta/models/<model>:generateContent, responseModalities IMAGE,
* imageConfig.aspectRatio); vision calls reuse the Chat Completions body on
* Google's OpenAI-compatible endpoint (/v1beta/openai/chat/completions).
* The key travels only in a header (x-goog-api-key or Authorization), never
* in a URL, and OPENAI_BASE_URL never redirects a Gemini call.
* Shapes: https://ai.google.dev/gemini-api/docs/image-generation and
* https://ai.google.dev/gemini-api/docs/openai (checked 2026-10-31).
*/
import { openaiUrl, resolveGeminiKeyInfo, describeKeySource, type ProviderKey, type ProviderName } from "./auth";
import { DESIGN_RECEIPTS, type ReceiptSink } from "./receipted-fetch";
import {
geminiAspectRatio,
geminiModel,
imageRequestBody,
modelRejectionHint,
REASONING_HEADROOM_TOKENS,
visionRequestBody,
type DesignCall,
} from "./models";
export const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta";
/** A caller may pass a bare OpenAI key (the pre-provider API) or a resolved provider. */
export type DesignAuth = string | ProviderKey;
export function asProvider(auth: DesignAuth): ProviderKey {
return typeof auth === "string" ? { provider: "openai", key: auth, source: "caller" } : auth;
}
export interface DesignRequest {
url: string;
init: { method: "POST"; headers: Record<string, string>; body: string };
receipt: ReceiptSink;
}
export interface ImageAsk {
prompt: string;
size: string;
quality: string;
/** Base64 PNGs sent with the prompt (Gemini only: the image to edit). */
images?: string[];
/** OpenAI only: continue a Responses API thread. */
previousResponseId?: string;
}
const post = (url: string, headers: Record<string, string>, body: string, provider: ProviderName): DesignRequest => ({
url,
init: { method: "POST", headers: { ...headers, "Content-Type": "application/json" }, body },
receipt: DESIGN_RECEIPTS[provider],
});
/** The request for one image call. Throws, before anything is sent, on a bad model override or size. */
export function imageRequest(auth: DesignAuth, ask: ImageAsk): DesignRequest {
const p = asProvider(auth);
if (p.provider === "openai") {
const extra = ask.previousResponseId ? { previous_response_id: ask.previousResponseId } : {};
return post(openaiUrl("responses"), { Authorization: `Bearer ${p.key}` },
imageRequestBody(ask.prompt, { size: ask.size, quality: ask.quality }, extra), "openai");
}
const parts: unknown[] = (ask.images ?? []).map(data => ({ inlineData: { mimeType: "image/png", data } }));
parts.push({ text: ask.prompt });
const body = JSON.stringify({
contents: [{ role: "user", parts }],
generationConfig: { responseModalities: ["IMAGE"], imageConfig: { aspectRatio: geminiAspectRatio(ask.size) } },
});
const url = `${GEMINI_API_BASE}/models/${encodeURIComponent(geminiModel("image"))}:generateContent`;
return post(url, { "x-goog-api-key": p.key }, body, "gemini");
}
/** The request for one vision call: the same Chat Completions body on either provider. */
export function visionRequest(auth: DesignAuth, messages: unknown[], visibleTokens: number, extra: Record<string, unknown> = {}): DesignRequest {
const p = asProvider(auth);
if (p.provider === "openai") {
return post(openaiUrl("chat/completions"), { Authorization: `Bearer ${p.key}` }, visionRequestBody(messages, visibleTokens, extra), "openai");
}
const body = JSON.stringify({
model: geminiModel("vision"),
messages,
max_completion_tokens: visibleTokens + REASONING_HEADROOM_TOKENS,
...extra,
});
return post(`${GEMINI_API_BASE}/openai/chat/completions`, { Authorization: `Bearer ${p.key}` }, body, "gemini");
}
/** A failure the caller must not pay to repeat (a safety block, a quota that will not refill soon). */
export class DesignImageError extends Error {
constructor(message: string, readonly retryable: boolean) {
super(message);
}
}
const GEMINI_BLOCK_REASONS = new Set(["SAFETY", "IMAGE_SAFETY", "PROHIBITED_CONTENT", "RECITATION", "BLOCKLIST", "SPII", "IMAGE_PROHIBITED_CONTENT", "IMAGE_RECITATION"]);
/**
* The image in a successful response, or null when it held none (OpenAI: no
* image_generation_call; Gemini: an empty answer with a normal finish). A
* Gemini safety block or text-only answer throws a non-retryable
* DesignImageError: asking again would buy the same refusal.
*/
export function readImage(auth: DesignAuth, data: any): { responseId: string; imageData: string } | null {
const p = asProvider(auth);
if (p.provider === "openai") {
const item = data?.output?.find((o: any) => o.type === "image_generation_call");
return item?.result ? { responseId: data.id, imageData: item.result } : null;
}
const blocked = data?.promptFeedback?.blockReason;
if (blocked) throw new DesignImageError(`Gemini blocked this brief (${blocked}). Fix: rephrase the brief.`, false);
const candidate = data?.candidates?.[0];
const parts: any[] = candidate?.content?.parts ?? [];
const images = parts.filter(part => {
const inline = part?.inlineData ?? part?.inline_data;
return typeof inline?.data === "string" && /^image\//.test(inline.mimeType ?? inline.mime_type ?? "");
});
const image = images.findLast(part => part?.thought !== true) ?? images.at(-1);
if (image) {
const inline = image.inlineData ?? image.inline_data;
const mime = inline.mimeType ?? inline.mime_type;
if (mime !== "image/png") throw new DesignImageError(`Gemini returned ${mime}, not a PNG; nothing was saved.`, false);
return { responseId: data.responseId || `gemini-${Date.now()}`, imageData: inline.data };
}
const finish = candidate?.finishReason;
if (finish && GEMINI_BLOCK_REASONS.has(finish)) throw new DesignImageError(`Gemini blocked this brief (${finish}). Fix: rephrase the brief.`, false);
const text = parts.map(part => part?.text).filter(Boolean).join(" ").trim();
if (text) throw new DesignImageError(`Gemini answered with text instead of an image: ${text.slice(0, 200)}`, false);
return null;
}
/** OpenAI's org-verification refusal (every OpenAI image model needs a verified organization). */
export function isOpenAiVerificationError(status: number, body: string): boolean {
return status === 403 && body.includes("organization must be verified");
}
/** The last line of an OpenAI org-verification error: how to reach Gemini instead. */
export function geminiSwitchFix(): string {
const gemini = resolveGeminiKeyInfo();
if (gemini) return `Fix: a Gemini key is configured (${describeKeySource("gemini", gemini)}); set GSTACK_DESIGN_PROVIDER=gemini to use it.`;
return "Fix: or use Gemini instead: get a free key at https://aistudio.google.com/apikey, run $D setup --provider gemini, then set GSTACK_DESIGN_PROVIDER=gemini.";
}
export interface ProviderFailure {
message: string;
retryable: boolean;
/** Gemini's RetryInfo delay for a recoverable 429, uncapped (the caller caps it). */
retryAfterMs?: number;
}
const retryDelayMs = (details: any[]): number | undefined => {
const delay = details.find(d => typeof d?.retryDelay === "string")?.retryDelay as string | undefined;
const seconds = delay ? Number.parseFloat(delay) : NaN;
return Number.isFinite(seconds) ? Math.max(0, seconds * 1000) : undefined;
};
/**
* A Gemini HTTP failure as a message whose last line says how to fix it,
* with whether repeating the request could succeed. Names the provider, the
* model and where the key came from, never the key.
*/
export function geminiFailure(auth: DesignAuth, status: number, body: string, call: DesignCall): ProviderFailure {
const p = asProvider(auth);
let error: any = {};
try { error = JSON.parse(body)?.error ?? {}; } catch { /* non-JSON body */ }
const message = String(error.message ?? body).slice(0, 300);
const details: any[] = Array.isArray(error.details) ? error.details : [];
const reasons = details.map(d => d?.reason).filter(Boolean).join(" ");
const model = geminiModel(call);
const head = `Gemini API error (${status}, model ${model}, key from ${p.source}): ${message}`;
if (/API_KEY_INVALID/.test(reasons) || /API key not valid/i.test(message)) {
return { message: `${head}\nFix: create a key at https://aistudio.google.com/apikey and run $D setup --provider gemini. See docs/troubleshooting.md#design-gemini-key-rejected`, retryable: false };
}
if (status === 429) {
const perDay = /PerDay/i.test(JSON.stringify(details)) || /limit: 0\b/.test(message);
if (perDay) {
return { message: `${head}\nFix: this key has no ${call} quota left today (free-tier keys may have none for image models); enable billing in Google AI Studio or set GSTACK_DESIGN_PROVIDER=openai. See docs/troubleshooting.md#design-gemini-quota`, retryable: false };
}
return { message: `Rate limited (429): ${message}`, retryable: true, retryAfterMs: retryDelayMs(details) };
}
if (status === 403 || /SERVICE_DISABLED|PERMISSION_DENIED/.test(reasons)) {
return { message: `${head}\nFix: enable the Generative Language API for this key's Google Cloud project, or set GSTACK_DESIGN_PROVIDER=openai. See docs/troubleshooting.md#design-gemini-key-rejected`, retryable: false };
}
if (status === 400 && /location is not supported/i.test(message)) {
return { message: `${head}\nFix: the Gemini API is not offered in your region; use OpenAI (set GSTACK_DESIGN_PROVIDER=openai).`, retryable: false };
}
if (status === 404) {
const variable = call === "image" ? "GSTACK_DESIGN_IMAGE_MODEL" : "GSTACK_DESIGN_MODEL";
return { message: `${head}\nFix: set ${variable} to a Gemini model your key can use, or unset it. See docs/troubleshooting.md#design-image-model-invalid`, retryable: false };
}
return { message: head, retryable: status >= 500 };
}
/** The suffix a vision caller appends to an HTTP error: OpenAI's model hint, or Gemini's mapped reason. */
export function visionErrorHint(auth: DesignAuth, status: number, body: string): string {
const p = asProvider(auth);
if (p.provider === "openai") return modelRejectionHint(status, body, "vision");
return ` — ${geminiFailure(p, status, body, "vision").message.replace(/\n/g, " ")}`;
}