/** * $D CLI persistence contract with the image API stubbed (no spend): paid * images are never overwritten, every saved image is reported, a local save * failure never buys a second image, and JSON is printed on every exit. */ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; const CLI = path.join(import.meta.dir, "..", "src", "cli.ts"); const RUN_FIELDS = ["requested", "saved", "selected", "failures", "recovered"]; let dir: string; let calls: string; let preload: string; const sessions: string[] = []; beforeEach(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), "design-cli-persist-")); calls = path.join(dir, "calls.log"); preload = path.join(dir, "stub.ts"); fs.writeFileSync(preload, `import { appendFileSync, existsSync, readFileSync } from "node:fs"; const seq = (name: string) => (process.env[name] || "").split(",").filter(Boolean); globalThis.fetch = (async (url: unknown) => { const pathname = new URL(String(url)).pathname; const log = process.env.CALLS!; const prior = existsSync(log) ? readFileSync(log, "utf8").split("\\n").filter(l => l === pathname).length : 0; appendFileSync(log, pathname + "\\n"); if (pathname === "/v1/responses") { const step = seq("IMAGES")[prior] ?? "ok"; if (step === "fail") return new Response("upstream unavailable", { status: 503 }); const result = step === "empty" ? "====" : Buffer.from("image " + prior).toString("base64"); return Response.json({ id: "resp-" + prior, output: [{ type: "image_generation_call", result }] }); } const step = seq("CHECKS")[prior] ?? "PASS"; if (step === "throw") throw Object.assign(new Error("The operation was aborted."), { name: "AbortError" }); return Response.json({ choices: [{ message: { content: step === "PASS" ? "PASS" : "FAIL: illegible title" } }] }); }) as typeof fetch; `); }); afterEach(() => { for (const s of sessions.splice(0)) fs.rmSync(s, { force: true }); fs.rmSync(dir, { recursive: true, force: true }); }); function run(args: string[], env: Record = {}) { const r = spawnSync(process.execPath, ["--no-env-file", "--preload", preload, CLI, ...args], { encoding: "utf8", timeout: 60_000, env: { PATH: process.env.PATH!, HOME: dir, GSTACK_HOME: path.join(dir, "state"), TMPDIR: dir, OPENAI_API_KEY: "fixture-not-a-real-key", CALLS: calls, ...env, }, }); const json = JSON.parse(r.stdout); for (const a of json.attempts ?? []) if (a.sessionFile) sessions.push(a.sessionFile); return { status: r.status, stderr: r.stderr, json }; } const apiCalls = () => fs.existsSync(calls) ? fs.readFileSync(calls, "utf8").split("\n").filter(l => l === "/v1/responses").length : 0; describe("never overwrite", () => { // Value: protects=a second generate into the same --output keeps the first image and reports the bumped path with the exact notice; // fails_when=generate writes with writeFileSync again or reports the requested path; why_new=#1529 had no CLI coverage; seam=none test("generate bumps, prints the notice and leaves the original bytes", () => { const out = path.join(dir, "mock.png"); const first = run(["generate", "--brief", "dashboard", "--output", out]); expect(first.status, first.stderr).toBe(0); expect(first.json.outputPath).toBe(out); const second = run(["generate", "--brief", "dashboard", "--output", out]); const bumped = path.join(dir, "mock-2.png"); expect(second.status, second.stderr).toBe(0); expect(second.json.outputPath).toBe(bumped); expect(second.json.saved).toEqual([bumped]); expect(second.stderr).toContain(`note: ${out} exists; saved to ${bumped} (existing file kept)`); expect(fs.readFileSync(out, "utf8")).toBe("image 0"); expect(fs.readFileSync(bumped, "utf8")).toBe("image 1"); for (const key of [...RUN_FIELDS, "outputPath", "attempts"]) expect(second.json).toHaveProperty(key); }); // Value: protects=a second variants round keeps round one, reports variant-X-2 paths, and both branches print one key set; // fails_when=variants overwrites or the --viewports branch drops count/errors/failures; why_new=#1529 overwrite path; seam=none test("variants (both branches) bump per file with one shared key set", () => { const outDir = path.join(dir, "designs"); const r1 = run(["variants", "--brief", "dashboard", "--count", "2", "--output-dir", outDir]); expect(r1.json.paths).toEqual([path.join(outDir, "variant-A.png"), path.join(outDir, "variant-B.png")]); const r2 = run(["variants", "--brief", "dashboard", "--count", "2", "--output-dir", outDir]); expect(r2.status, r2.stderr).toBe(0); expect(r2.json.paths).toEqual([path.join(outDir, "variant-A-2.png"), path.join(outDir, "variant-B-2.png")]); expect(r2.stderr).toContain(`note: ${path.join(outDir, "variant-A.png")} exists; saved to ${path.join(outDir, "variant-A-2.png")} (existing file kept)`); expect(fs.readFileSync(path.join(outDir, "variant-A.png"), "utf8")).toBe("image 0"); run(["variants", "--brief", "dashboard", "--viewports", "desktop", "--output-dir", outDir]); const again = run(["variants", "--brief", "dashboard", "--viewports", "desktop", "--output-dir", outDir]); expect(again.json.paths).toEqual([path.join(outDir, "responsive-desktop-2.png")]); expect(fs.readFileSync(path.join(outDir, "responsive-desktop.png"), "utf8")).toBe("image 4"); const keys = (o: object) => Object.keys(o).filter(k => k !== "viewports").sort(); expect(keys(again.json)).toEqual(keys(r2.json)); expect(again.json).toMatchObject({ count: 1, succeeded: 1, failed: 0, errors: [], failures: [], viewports: ["desktop"] }); for (const key of [...RUN_FIELDS, "outputDir", "count", "succeeded", "failed", "paths", "errors"]) expect(r2.json).toHaveProperty(key); }); }); describe("accounting and exit codes", () => { // Value: protects=each --retry attempt is its own saved, counted image; fails_when=retries overwrite one --output or attempts drop rejected images; // why_new=the retry premise was wrong in #1737; seam=none test("generate --check --retry keeps the rejected and the passing image", () => { const out = path.join(dir, "retry.png"); const r = run(["generate", "--brief", "dashboard", "--output", out, "--check", "--retry", "1"], { CHECKS: "FAIL,PASS" }); expect(r.status, r.stderr).toBe(0); const second = path.join(dir, "retry-2.png"); expect(r.json.saved).toEqual([out, second]); expect(r.json.attempts.map((a: any) => [a.path, a.check.pass])).toEqual([[out, false], [second, true]]); expect(r.json.outputPath).toBe(second); expect(fs.existsSync(out) && fs.existsSync(second)).toBe(true); }); // Value: protects=an interrupted retry exits 3 with JSON listing the earlier saved attempt; fails_when=the error escapes as exit 1 without JSON; // why_new=new exit contract; seam=none test("interrupted retry exits 3 and still prints every saved path", () => { const out = path.join(dir, "retry.png"); const r = run(["generate", "--brief", "dashboard", "--output", out, "--check", "--retry", "2"], { CHECKS: "FAIL", IMAGES: "ok,fail" }); expect(r.status).toBe(3); expect(r.json).toMatchObject({ outputPath: null, selected: null, saved: [out], requested: 1 }); expect(r.json.failures).toHaveLength(1); expect(r.json.failures[0].reason).toContain("503"); }); // Value: protects=a quality-check timeout after a save still prints partial JSON and exits 3; fails_when=a check exception loses the saved path; // why_new=DX contract case; seam=none test("quality-check timeout after a save exits 3 with partial JSON", () => { const out = path.join(dir, "checked.png"); const r = run(["generate", "--brief", "dashboard", "--output", out, "--check"], { CHECKS: "throw" }); expect(r.status).toBe(3); expect(r.json.saved).toEqual([out]); expect(r.json.failures[0].reason).toContain("aborted"); }); // Value: protects=zero saved images exit 2 with JSON naming every failure; fails_when=all-fail exits 0 or prints nothing; // why_new=skills route on this code; seam=none test("variants with zero successes exits 2 with JSON", () => { const outDir = path.join(dir, "designs"); const r = run(["variants", "--brief", "dashboard", "--count", "2", "--output-dir", outDir], { IMAGES: "fail,fail" }); expect(r.status).toBe(2); expect(r.json).toMatchObject({ requested: 2, saved: [], paths: [], errors: ["variant-A.png", "variant-B.png"] }); expect(r.json.failures.map((f: any) => path.basename(f.file))).toEqual(["variant-A.png", "variant-B.png"]); }); // Value: protects=a partial batch exits 0 and names the failed basename; fails_when=one failure fails the whole round; // why_new=partial accounting is new; seam=none test("partial variants batch exits 0", () => { const outDir = path.join(dir, "designs"); const r = run(["variants", "--brief", "dashboard", "--count", "2", "--output-dir", outDir], { IMAGES: "ok,fail" }); expect(r.status).toBe(0); expect(r.json).toMatchObject({ succeeded: 1, failed: 1, paths: [path.join(outDir, "variant-A.png")], errors: ["variant-B.png"] }); }); // Value: protects=an empty image from the API is a failure, never a 0-byte PNG; fails_when=persistImage writes empty bytes; // why_new=CEO empty-b64 warning; seam=none test("empty image bytes exit 2 with no file", () => { const out = path.join(dir, "empty.png"); const r = run(["generate", "--brief", "dashboard", "--output", out], { IMAGES: "empty" }); expect(r.status).toBe(2); expect(fs.existsSync(out)).toBe(false); expect(r.json.failures[0].reason).toContain("empty image"); }); // Value: protects=a missing API key still prints the run JSON before exiting non-zero; fails_when=the key check exits the process directly; // why_new=JSON on every exit path; seam=none test("missing API key prints JSON and exits 2", () => { const r = run(["evolve", "--screenshot", path.join(dir, "none.png"), "--brief", "calmer", "--output", path.join(dir, "e.png")], { OPENAI_API_KEY: "" }); expect(r.status).toBe(2); expect(r.json.failures[0].reason).toContain("No image API key found (OpenAI or Gemini)"); for (const key of [...RUN_FIELDS, "outputPath", "attempts"]) expect(r.json).toHaveProperty(key); }); }); describe("no re-purchase on a local save failure", () => { // Value: protects=a write failure after a paid response makes exactly one API call per variant and keeps a recovery copy; // fails_when=the save sits inside the 429 retry loop again; why_new=Codex DX finding 1; seam=none test("variants (both branches): one API call, recovery copy, exit 2", () => { const blocker = path.join(dir, "blocker"); fs.writeFileSync(blocker, "a file, not a directory"); for (const extra of [["--count", "1"], ["--viewports", "desktop"]]) { const before = apiCalls(); const r = run(["variants", "--brief", "dashboard", ...extra, "--output-dir", path.join(blocker, "designs")]); expect(r.status).toBe(2); expect(apiCalls() - before).toBe(1); expect(r.stderr).not.toContain("Rate limited"); expect(r.json.recovered).toHaveLength(1); expect(fs.existsSync(r.json.recovered[0].path)).toBe(true); expect(r.json.failures[0].reason).toContain("cannot save paid image to"); } }); // Value: protects=iterate makes one claim per run, and a save failure never triggers the fallback purchase; // fails_when=the write moves back inside the threaded try; why_new=eng loop restructure; seam=none test("iterate: threaded failure claims once; save failure buys nothing more", () => { const gen = run(["generate", "--brief", "dashboard", "--output", path.join(dir, "base.png")]); const session = gen.json.sessionFile; const out = path.join(dir, "refined.png"); const before = apiCalls(); const fallback = run(["iterate", "--session", session, "--feedback", "bigger", "--output", out], { IMAGES: "ok,fail,ok" }); expect(fallback.status, fallback.stderr).toBe(0); expect(apiCalls() - before).toBe(2); expect(fallback.json.saved).toEqual([out]); expect(fs.readdirSync(dir).filter(f => f.startsWith("refined"))).toEqual(["refined.png"]); const blocker = path.join(dir, "blocker"); fs.writeFileSync(blocker, "file"); const mid = apiCalls(); const failed = run(["iterate", "--session", session, "--feedback", "bigger", "--output", path.join(blocker, "x.png")]); expect(failed.status).toBe(2); expect(apiCalls() - mid).toBe(1); expect(failed.stderr).not.toContain("Falling back"); expect(failed.json.recovered).toHaveLength(1); for (const key of [...RUN_FIELDS, "outputPath", "attempts", "sessionFile", "iteration"]) expect(failed.json).toHaveProperty(key); }); });