mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-06 08:54:04 +01:00
Merge remote-tracking branch 'origin/garrytan/opus55fu-wip-H5' into garrytan/opus55-followups
This commit is contained in:
commit
bb2ca35f0d
44 files changed
+942
-215
No files matched your search
@@ -224,7 +224,7 @@ Re-run: copy RESTORE_PATH's bytes to SOURCE_PLAN, then /autoplan.
|
||||
### Step 2: Read context
|
||||
|
||||
- Read CLAUDE.md, TODOS.md, git log -30, git diff against the base branch --stat
|
||||
- Discover design docs: `ls -t ~/.gstack/projects/$SLUG/*-design-*.md 2>/dev/null | head -1`
|
||||
- Discover design docs: `~/.claude/skills/gstack/bin/gstack-design-doc-find "$SLUG" "$BRANCH"` (prints the doc path, or nothing)
|
||||
- Detect UI scope: grep the plan for view/rendering terms (component, screen, form,
|
||||
button, modal, layout, dashboard, sidebar, nav, dialog). Require 2+ matches. Exclude
|
||||
false positives ("page" alone, "UI" in acronyms).
|
||||
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
# gstack-design-doc-find — print the design doc a plan review should read.
|
||||
#
|
||||
# Usage: gstack-design-doc-find <slug> <branch>
|
||||
#
|
||||
# Prints one absolute path, or nothing when no design doc exists. Exit 0 either
|
||||
# way; exit 2 on a usage error.
|
||||
#
|
||||
# Precedence:
|
||||
# 1. newest $GSTACK_STATE_ROOT/projects/<slug>/*-<branch>-design-*.md
|
||||
# 2. else newest $GSTACK_STATE_ROOT/projects/<slug>/*-design-*.md
|
||||
# 3. a repo doc ($TOP/DESIGN.md, else newest $TOP/docs/designs/*.md) wins when
|
||||
# no state-root doc exists or it is newer than the state-root doc.
|
||||
# office-hours writes both copies; the committed copy is what teammates
|
||||
# see, but a stale repo doc never shadows a newer private session.
|
||||
set -u
|
||||
|
||||
if [ "$#" -ne 2 ] || [ -z "$1" ]; then
|
||||
echo "usage: gstack-design-doc-find <slug> <branch>" >&2
|
||||
exit 2
|
||||
fi
|
||||
SLUG=$1
|
||||
BRANCH=$2
|
||||
|
||||
eval "$("$(dirname "$0")/gstack-paths")"; : "${GSTACK_STATE_ROOT:?gstack-paths failed; reinstall with ./setup or /gstack-upgrade}"
|
||||
|
||||
newest() {
|
||||
ls -t "$@" 2>/dev/null | head -1
|
||||
}
|
||||
|
||||
PROJECT_DIR="$GSTACK_STATE_ROOT/projects/$SLUG"
|
||||
LOCALDOC=$(newest "$PROJECT_DIR"/*-"$BRANCH"-design-*.md)
|
||||
[ -z "$LOCALDOC" ] && LOCALDOC=$(newest "$PROJECT_DIR"/*-design-*.md)
|
||||
|
||||
REPODOC=""
|
||||
TOP=$(git rev-parse --show-toplevel 2>/dev/null || true)
|
||||
if [ -n "$TOP" ]; then
|
||||
if [ -f "$TOP/DESIGN.md" ]; then
|
||||
REPODOC="$TOP/DESIGN.md"
|
||||
else
|
||||
REPODOC=$(newest "$TOP"/docs/designs/*.md)
|
||||
fi
|
||||
fi
|
||||
|
||||
DESIGN="$LOCALDOC"
|
||||
if [ -n "$REPODOC" ] && { [ -z "$LOCALDOC" ] || [ "$REPODOC" -nt "$LOCALDOC" ]; }; then
|
||||
DESIGN="$REPODOC"
|
||||
fi
|
||||
[ -n "$DESIGN" ] && printf '%s\n' "$DESIGN"
|
||||
exit 0
|
||||
@@ -18,7 +18,6 @@ allowed-tools:
|
||||
- Read
|
||||
- Glob
|
||||
- Grep
|
||||
- Agent
|
||||
- AskUserQuestion
|
||||
gbrain:
|
||||
schema: 1
|
||||
@@ -227,8 +226,8 @@ them, not one team on different days.
|
||||
|
||||
Use AskUserQuestion to confirm before spending API credits:
|
||||
|
||||
> "These are the {N} directions I'll generate. Each takes ~60s, but I'll run them all
|
||||
> in parallel so total time is ~60 seconds regardless of count."
|
||||
> "These are the {N} directions I'll generate. They run in parallel, so the whole set
|
||||
> typically takes about 60 seconds."
|
||||
|
||||
Options:
|
||||
- A) Generate all {N} — looks good
|
||||
@@ -258,57 +257,43 @@ console.log("GSTACK_STEP_OK");
|
||||
Then `cp "<ASIDE_DIR>/current.png" "$_DESIGN_DIR/current.png"` and Read it so the user
|
||||
sees what you're evolving from.
|
||||
|
||||
**Launch N Agent subagents in a single message** (parallel execution). Use the Agent
|
||||
tool with `subagent_type: "general-purpose"` and `run_in_background: false` for each
|
||||
variant (parallel foreground calls in one message still run concurrently, and the
|
||||
comparison board needs every variant's result). Each agent is independent and handles
|
||||
its own generation, quality check, verification, and retry.
|
||||
**Generate every variant with one `$D variants --briefs-file` call.** Write one entry per
|
||||
confirmed concept to `$_DESIGN_DIR/briefs.json`: a JSON array of `{"brief": "<the full
|
||||
variant-specific brief>"}` objects, in concept order (A, B, C, ...), at most 7. When
|
||||
evolving, add `"screenshot": "<_DESIGN_DIR>/current.png"` to every entry. Then run this
|
||||
Bash call with `timeout: 600000` and wait for it to return:
|
||||
|
||||
**Important: $D path propagation.** The `$D` variable from DESIGN SETUP is a shell
|
||||
variable that agents do NOT inherit. Substitute the resolved absolute path (from the
|
||||
`DESIGN_READY: /path/to/design` output in Step 0) into each agent prompt.
|
||||
|
||||
**Agent prompt template** (one per variant, substitute all `{...}` values):
|
||||
|
||||
```
|
||||
Generate a design variant and save it.
|
||||
|
||||
Design binary: {absolute path to $D binary}
|
||||
Brief: {the full variant-specific brief for this direction}
|
||||
Output: /tmp/variant-{letter}.png
|
||||
Final location: {_DESIGN_DIR absolute path}/variant-{letter}.png
|
||||
|
||||
Steps:
|
||||
1. Run: {$D path} generate --brief "{brief}" --output /tmp/variant-{letter}.png
|
||||
2. If the command fails with a rate limit error (429 or "rate limit"), wait 5 seconds
|
||||
and retry. Up to 3 retries.
|
||||
3. If the output file is missing or empty after the command succeeds, retry once.
|
||||
4. Copy: cp /tmp/variant-{letter}.png {_DESIGN_DIR}/variant-{letter}.png
|
||||
5. Quality check: {$D path} check --image {_DESIGN_DIR}/variant-{letter}.png --brief "{brief}"
|
||||
If quality check fails, retry generation once.
|
||||
6. Verify: ls -lh {_DESIGN_DIR}/variant-{letter}.png
|
||||
7. Report exactly one of:
|
||||
VARIANT_{letter}_DONE: {file size}
|
||||
VARIANT_{letter}_FAILED: {error description}
|
||||
VARIANT_{letter}_RATE_LIMITED: exhausted retries
|
||||
```bash
|
||||
_VARIANT_TMP=$(mktemp -d /tmp/gstack-variants-XXXXXX)
|
||||
"$D" variants --briefs-file "$_DESIGN_DIR/briefs.json" --output-dir "$_VARIANT_TMP"; _VARIANTS_EXIT=$?
|
||||
cp "$_VARIANT_TMP"/variant-*.png "$_DESIGN_DIR"/ 2>/dev/null
|
||||
echo "VARIANTS_EXIT=$_VARIANTS_EXIT"
|
||||
```
|
||||
|
||||
For the evolve path, replace step 1 with:
|
||||
```
|
||||
{$D path} evolve --screenshot {_DESIGN_DIR}/current.png --brief "{brief}" --output /tmp/variant-{letter}.png
|
||||
```
|
||||
The command starts the variants 1.5s apart, retries rate limits with backoff, regenerates a
|
||||
missing or empty image once, runs the vision check on each image and regenerates once
|
||||
when it fails, and starts no new work after 9 minutes. It prints one
|
||||
`VARIANT_<letter>_DONE`, `_FAILED` or `_RATE_LIMITED` line per variant on stderr and a JSON
|
||||
summary on stdout: per variant `path`, `operation`, `status`, `error`, `retryable` and
|
||||
`check.status` (`pass`, `fail` or `skipped`). Exit 0 means every variant was generated, 3
|
||||
means some failed, 1 means none were generated or the briefs file was invalid (the error
|
||||
names the entry and field; fix it and rerun).
|
||||
|
||||
**Generate to `/tmp/`, then `cp`:** in sandboxed sessions `$D generate --output` under
|
||||
`~/.gstack/` can abort ("The operation was aborted"), while `/tmp/` works.
|
||||
**Generate to `/tmp/`, then `cp`:** in sandboxed sessions `$D` output under `~/.gstack/`
|
||||
can abort ("The operation was aborted"), while `/tmp/` works.
|
||||
|
||||
### Step 3d: Results
|
||||
|
||||
After all agents complete:
|
||||
After the command returns:
|
||||
|
||||
1. Read each generated PNG inline (Read tool) so the user sees all variants at once.
|
||||
2. Report status: "All {N} variants generated in ~{actual time}. {successes} succeeded,
|
||||
{failures} failed."
|
||||
3. For any failures: report explicitly with the error. Do NOT silently skip.
|
||||
{failures} failed." A `skipped` check is missing automated coverage, not a pass: say so.
|
||||
3. For any failures: report explicitly with the error. Do NOT silently skip. Rerun each
|
||||
failed variant with `retryable: true` once, using its own operation and brief:
|
||||
`"$D" generate --brief "<brief>" --output /tmp/variant-<letter>.png`, or for a screenshot
|
||||
entry `"$D" evolve --screenshot "$_DESIGN_DIR/current.png" --brief "<brief>" --output /tmp/variant-<letter>.png`,
|
||||
then `cp` it into `$_DESIGN_DIR`.
|
||||
4. If zero variants succeeded: fall back to sequential generation (one at a time with
|
||||
`$D generate`, showing each as it lands). Tell the user: "Parallel generation failed
|
||||
(likely rate limiting). Falling back to sequential..."
|
||||
|
||||
+18
-8
@@ -8,20 +8,30 @@ import { requireApiKey } from "./auth";
|
||||
import { receiptedFetch } from "./receipted-fetch";
|
||||
import { modelRejectionHint, visionRequestBody } from "./models";
|
||||
|
||||
/** `pass` stays true when the check could not run; `status: "skipped"` marks that
|
||||
* no automated validation happened, so callers never count it as validated. */
|
||||
export interface CheckResult {
|
||||
pass: boolean;
|
||||
status: "pass" | "fail" | "skipped";
|
||||
issues: string;
|
||||
}
|
||||
|
||||
export interface CheckOptions {
|
||||
apiKey?: string;
|
||||
fetchFn?: typeof globalThis.fetch;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check a generated mockup against the original brief.
|
||||
*/
|
||||
export async function checkMockup(imagePath: string, brief: string): Promise<CheckResult> {
|
||||
const apiKey = requireApiKey();
|
||||
export async function checkMockup(imagePath: string, brief: string, opts: CheckOptions = {}): Promise<CheckResult> {
|
||||
const apiKey = opts.apiKey ?? requireApiKey();
|
||||
const imageData = fs.readFileSync(imagePath).toString("base64");
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 60_000);
|
||||
const signal = opts.signal ? AbortSignal.any([controller.signal, opts.signal]) : controller.signal;
|
||||
|
||||
try {
|
||||
const response = await receiptedFetch("check-screenshot-request", "https://api.openai.com/v1/chat/completions", {
|
||||
@@ -56,30 +66,30 @@ export async function checkMockup(imagePath: string, brief: string): Promise<Che
|
||||
},
|
||||
],
|
||||
}], 200),
|
||||
signal: controller.signal,
|
||||
});
|
||||
signal,
|
||||
}, opts.fetchFn);
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
if (response.status === 403 && error.includes("organization must be verified")) {
|
||||
console.error("OpenAI organization verification required. Go to https://platform.openai.com/settings/organization to verify.");
|
||||
return { pass: true, issues: "OpenAI org not verified — vision check skipped" };
|
||||
return { pass: true, status: "skipped", issues: "OpenAI org not verified — vision check skipped" };
|
||||
}
|
||||
// Non-blocking: if vision check fails, default to PASS with warning
|
||||
console.error(`Vision check API error (${response.status}): ${error}${modelRejectionHint(response.status, error, "vision")}`);
|
||||
return { pass: true, issues: "Vision check unavailable — skipped" };
|
||||
return { pass: true, status: "skipped", issues: "Vision check unavailable — skipped" };
|
||||
}
|
||||
|
||||
const data = await response.json() as any;
|
||||
const content = data.choices?.[0]?.message?.content?.trim() || "";
|
||||
|
||||
if (content.startsWith("PASS")) {
|
||||
return { pass: true, issues: "" };
|
||||
return { pass: true, status: "pass", issues: "" };
|
||||
}
|
||||
|
||||
// Extract issues after "FAIL:"
|
||||
const issues = content.replace(/^FAIL:\s*/i, "").trim();
|
||||
return { pass: false, issues: issues || content };
|
||||
return { pass: false, status: "fail", issues: issues || content };
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
|
||||
+2
-1
@@ -195,9 +195,10 @@ async function main(): Promise<void> {
|
||||
break;
|
||||
|
||||
case "variants":
|
||||
await variants({
|
||||
process.exitCode = await variants({
|
||||
brief: flags.brief as string,
|
||||
briefFile: flags["brief-file"] as string,
|
||||
briefsFile: flags["briefs-file"] as string,
|
||||
// #2032: pass the RAW flag through — variants() normalizes at its
|
||||
// consumption site (a pre-parseInt here would silently truncate "3.7").
|
||||
count: flags.count,
|
||||
|
||||
@@ -20,9 +20,9 @@ export const COMMANDS = new Map<string, {
|
||||
flags: ["--brief", "--brief-file", "--output", "--check", "--retry", "--size", "--quality"],
|
||||
}],
|
||||
["variants", {
|
||||
description: "Generate N design variants from a brief",
|
||||
usage: "variants --brief \"...\" --count 3 --output-dir /path/",
|
||||
flags: ["--brief", "--brief-file", "--count", "--output-dir", "--size", "--quality", "--viewports"],
|
||||
description: "Generate N design variants from a brief, or one variant per entry of a briefs file",
|
||||
usage: "variants --brief \"...\" --count 3 --output-dir /path/ | variants --briefs-file briefs.json --output-dir /path/ (briefs.json: [{\"brief\": \"Calm dashboard...\"}, {\"brief\": \"Bolder header...\", \"screenshot\": \"current.png\"}], 1-7 entries; exit 0 all generated, 3 some failed, 1 none or invalid input)",
|
||||
flags: ["--brief", "--brief-file", "--briefs-file", "--count", "--output-dir", "--size", "--quality", "--viewports"],
|
||||
}],
|
||||
["iterate", {
|
||||
description: "Iterate on an existing mockup with feedback",
|
||||
|
||||
+27
-16
@@ -39,19 +39,7 @@ export async function evolve(options: EvolveOptions): Promise<void> {
|
||||
console.error(` Analyzed current design: ${analysis.slice(0, 100)}...`);
|
||||
|
||||
// Step 2: Generate evolved version using analysis + brief
|
||||
const evolvedPrompt = [
|
||||
"Generate a pixel-perfect UI mockup that is an improved version of an existing design.",
|
||||
"",
|
||||
"CURRENT DESIGN (what exists now):",
|
||||
analysis,
|
||||
"",
|
||||
"REQUESTED CHANGES:",
|
||||
options.brief,
|
||||
"",
|
||||
"Generate a new mockup that keeps the existing layout structure but applies the requested changes.",
|
||||
"The result should look like a real production UI. All text must be readable.",
|
||||
"1536x1024 pixels.",
|
||||
].join("\n");
|
||||
const evolvedPrompt = evolvePrompt(analysis, options.brief);
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 240_000);
|
||||
@@ -103,12 +91,35 @@ export async function evolve(options: EvolveOptions): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
/** The image prompt for an evolved mockup: the analyzed current design plus the requested changes. */
|
||||
export function evolvePrompt(analysis: string, brief: string): string {
|
||||
return [
|
||||
"Generate a pixel-perfect UI mockup that is an improved version of an existing design.",
|
||||
"",
|
||||
"CURRENT DESIGN (what exists now):",
|
||||
analysis,
|
||||
"",
|
||||
"REQUESTED CHANGES:",
|
||||
brief,
|
||||
"",
|
||||
"Generate a new mockup that keeps the existing layout structure but applies the requested changes.",
|
||||
"The result should look like a real production UI. All text must be readable.",
|
||||
"1536x1024 pixels.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze a screenshot to produce a detailed description for re-generation.
|
||||
*/
|
||||
async function analyzeScreenshot(apiKey: string, imageBase64: string): Promise<string> {
|
||||
export async function analyzeScreenshot(
|
||||
apiKey: string,
|
||||
imageBase64: string,
|
||||
fetchFn: typeof globalThis.fetch = globalThis.fetch,
|
||||
batchSignal?: AbortSignal,
|
||||
): Promise<string> {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 30_000);
|
||||
const signal = batchSignal ? AbortSignal.any([controller.signal, batchSignal]) : controller.signal;
|
||||
|
||||
try {
|
||||
const response = await receiptedFetch("evolve-screenshot-analysis-request", "https://api.openai.com/v1/chat/completions", {
|
||||
@@ -130,8 +141,8 @@ async function analyzeScreenshot(apiKey: string, imageBase64: string): Promise<s
|
||||
},
|
||||
],
|
||||
}], 400),
|
||||
signal: controller.signal,
|
||||
});
|
||||
signal,
|
||||
}, fetchFn);
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
|
||||
@@ -135,7 +135,7 @@ export async function generate(options: GenerateOptions): Promise<GenerateResult
|
||||
lastResult.checkResult = checkResult;
|
||||
|
||||
if (checkResult.pass) {
|
||||
console.error(`Quality check: PASS`);
|
||||
console.error(checkResult.status === "skipped" ? `Quality check: SKIPPED — ${checkResult.issues}` : `Quality check: PASS`);
|
||||
break;
|
||||
} else {
|
||||
console.error(`Quality check: FAIL — ${checkResult.issues}`);
|
||||
|
||||
+217
-16
@@ -11,10 +11,14 @@ import { receiptedFetch } from "./receipted-fetch";
|
||||
import { imageRequestBody, modelRejectionHint } from "./models";
|
||||
import { parseBrief } from "./brief";
|
||||
import { normalizeIntFlag } from "./flag-utils";
|
||||
import { checkMockup, type CheckResult } from "./check";
|
||||
import { analyzeScreenshot, evolvePrompt } from "./evolve";
|
||||
|
||||
export interface VariantsOptions {
|
||||
brief?: string;
|
||||
briefFile?: string;
|
||||
/** JSON array of { brief, screenshot? }, one entry per variant. */
|
||||
briefsFile?: string;
|
||||
/**
|
||||
* Raw CLI flag value or a number. Normalized inside variants() (#2032):
|
||||
* nonsense errors loudly; above STYLE_VARIATIONS.length clamps with a
|
||||
@@ -37,11 +41,35 @@ const STYLE_VARIATIONS = [
|
||||
"Use a playful, modern style with asymmetric layout and unexpected color accents.",
|
||||
];
|
||||
|
||||
export interface VariantResult {
|
||||
path: string;
|
||||
success: boolean;
|
||||
error?: string;
|
||||
/** Exhausted its 429 retries. */
|
||||
rateLimited?: boolean;
|
||||
/** A rerun of the same operation could succeed (rate limit, timeout, deadline, 5xx, empty output). */
|
||||
retryable?: boolean;
|
||||
}
|
||||
|
||||
const DEADLINE_ERROR = "Batch deadline reached";
|
||||
|
||||
/** Resolves after `ms`, or as soon as `signal` aborts. */
|
||||
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
return new Promise(resolve => {
|
||||
if (signal?.aborted) return resolve();
|
||||
const done = () => { clearTimeout(timer); signal?.removeEventListener("abort", done); resolve(); };
|
||||
const timer = setTimeout(done, ms);
|
||||
signal?.addEventListener("abort", done, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a single variant with retry on 429.
|
||||
*
|
||||
* Exported for testability. Pass `fetchFn` to inject a stubbed fetch in tests;
|
||||
* production code uses the global fetch by default.
|
||||
* production code uses the global fetch by default. `signal` (the batch
|
||||
* deadline) stops backoff waits, the request and its body read; once it has
|
||||
* aborted no new attempt starts.
|
||||
*/
|
||||
export async function generateVariant(
|
||||
apiKey: string,
|
||||
@@ -50,7 +78,8 @@ export async function generateVariant(
|
||||
size: string,
|
||||
quality: string,
|
||||
fetchFn: typeof globalThis.fetch = globalThis.fetch,
|
||||
): Promise<{ path: string; success: boolean; error?: string }> {
|
||||
signal?: AbortSignal,
|
||||
): Promise<VariantResult> {
|
||||
const maxRetries = 3;
|
||||
const MAX_RETRY_AFTER_MS = 60_000; // cap honored Retry-After to bound stalls
|
||||
let lastError = "";
|
||||
@@ -59,7 +88,7 @@ export async function generateVariant(
|
||||
try {
|
||||
body = imageRequestBody(prompt, { size, quality });
|
||||
} catch (err: any) {
|
||||
return { path: outputPath, success: false, error: err.message };
|
||||
return { path: outputPath, success: false, error: err.message, retryable: false };
|
||||
}
|
||||
|
||||
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
||||
@@ -67,9 +96,10 @@ export async function generateVariant(
|
||||
// Exponential backoff: 2s, 4s, 8s
|
||||
const delay = Math.pow(2, attempt) * 1000;
|
||||
console.error(` Rate limited, retrying in ${delay / 1000}s...`);
|
||||
await new Promise(r => setTimeout(r, delay));
|
||||
await sleep(delay, signal);
|
||||
}
|
||||
skipLeadingDelay = false;
|
||||
if (signal?.aborted) return { path: outputPath, success: false, error: DEADLINE_ERROR, retryable: true };
|
||||
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 240_000);
|
||||
@@ -82,12 +112,11 @@ export async function generateVariant(
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body,
|
||||
signal: controller.signal,
|
||||
signal: signal ? AbortSignal.any([controller.signal, signal]) : controller.signal,
|
||||
}, fetchFn);
|
||||
|
||||
clearTimeout(timeout);
|
||||
|
||||
if (response.status === 429) {
|
||||
clearTimeout(timeout);
|
||||
lastError = "Rate limited (429)";
|
||||
const retryAfter = response.headers.get("retry-after");
|
||||
if (retryAfter) {
|
||||
@@ -105,7 +134,7 @@ export async function generateVariant(
|
||||
}
|
||||
if (waitMs !== null) {
|
||||
if (waitMs > 0) {
|
||||
await new Promise(resolve => setTimeout(resolve, waitMs));
|
||||
await sleep(waitMs, signal);
|
||||
}
|
||||
// Honored Retry-After (incl. 0 / past date "retry now") — skip the
|
||||
// next iteration's leading exponential sleep so we don't double-wait.
|
||||
@@ -117,17 +146,19 @@ export async function generateVariant(
|
||||
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
clearTimeout(timeout);
|
||||
if (response.status === 403 && error.includes("organization must be verified")) {
|
||||
return { path: outputPath, success: false, error: "OpenAI organization verification required. Go to https://platform.openai.com/settings/organization to verify." };
|
||||
return { path: outputPath, success: false, error: "OpenAI organization verification required. Go to https://platform.openai.com/settings/organization to verify.", retryable: false };
|
||||
}
|
||||
return { path: outputPath, success: false, error: `API error (${response.status}): ${error.slice(0, 200)}${modelRejectionHint(response.status, error, "image")}` };
|
||||
return { path: outputPath, success: false, error: `API error (${response.status}): ${error.slice(0, 200)}${modelRejectionHint(response.status, error, "image")}`, retryable: response.status >= 500 };
|
||||
}
|
||||
|
||||
const data = await response.json() as any;
|
||||
clearTimeout(timeout);
|
||||
const imageItem = data.output?.find((item: any) => item.type === "image_generation_call");
|
||||
|
||||
if (!imageItem?.result) {
|
||||
return { path: outputPath, success: false, error: "No image data in response" };
|
||||
return { path: outputPath, success: false, error: "No image data in response", retryable: true };
|
||||
}
|
||||
|
||||
fs.writeFileSync(outputPath, Buffer.from(imageItem.result, "base64"));
|
||||
@@ -135,19 +166,188 @@ export async function generateVariant(
|
||||
} catch (err: any) {
|
||||
clearTimeout(timeout);
|
||||
if (err.name === "AbortError") {
|
||||
return { path: outputPath, success: false, error: "Timeout (240s)" };
|
||||
return { path: outputPath, success: false, error: signal?.aborted ? DEADLINE_ERROR : "Timeout (240s)", retryable: true };
|
||||
}
|
||||
lastError = err.message;
|
||||
}
|
||||
}
|
||||
|
||||
return { path: outputPath, success: false, error: lastError };
|
||||
return { path: outputPath, success: false, error: lastError, rateLimited: lastError === "Rate limited (429)", retryable: true };
|
||||
}
|
||||
|
||||
export interface BriefEntry {
|
||||
brief: string;
|
||||
screenshot?: string;
|
||||
}
|
||||
|
||||
/** One entry per variant; the ceiling matches the built-in style directions. */
|
||||
export const MAX_BRIEFS = STYLE_VARIATIONS.length;
|
||||
/** Whole-batch deadline; the skill's Bash call allows 600s. */
|
||||
export const BATCH_DEADLINE_MS = 540_000;
|
||||
const STAGGER_MS = 1500;
|
||||
|
||||
/**
|
||||
* Read and validate a --briefs-file before any billable call. Errors name the
|
||||
* file, the entry index and the field.
|
||||
*/
|
||||
export function readBriefsFile(file: string): BriefEntry[] {
|
||||
const fail = (problem: string): never => { throw new Error(`--briefs-file ${file}: ${problem}`); };
|
||||
let raw: unknown;
|
||||
try {
|
||||
raw = JSON.parse(fs.readFileSync(file, "utf-8"));
|
||||
} catch (err: any) {
|
||||
return fail(`cannot read a JSON array (${err.message})`);
|
||||
}
|
||||
if (!Array.isArray(raw) || raw.length < 1 || raw.length > MAX_BRIEFS) {
|
||||
return fail(`expected a JSON array of 1 to ${MAX_BRIEFS} entries like [{"brief": "...", "screenshot": "optional.png"}]`);
|
||||
}
|
||||
return raw.map((entry: any, index) => {
|
||||
if (!entry || typeof entry !== "object" || Array.isArray(entry)) fail(`entry ${index}: expected an object with a "brief" field`);
|
||||
const unknown = Object.keys(entry).find(key => key !== "brief" && key !== "screenshot");
|
||||
if (unknown) fail(`entry ${index}: unknown field "${unknown}" (allowed: brief, screenshot)`);
|
||||
if (typeof entry.brief !== "string" || !entry.brief.trim()) fail(`entry ${index}: field "brief" must be a non-empty string`);
|
||||
if (entry.screenshot === undefined) return { brief: entry.brief };
|
||||
if (typeof entry.screenshot !== "string" || !entry.screenshot) fail(`entry ${index}: field "screenshot" must be a non-empty string`);
|
||||
let readable = false;
|
||||
try {
|
||||
fs.accessSync(entry.screenshot, fs.constants.R_OK);
|
||||
readable = fs.statSync(entry.screenshot).isFile();
|
||||
} catch {}
|
||||
if (!readable) fail(`entry ${index}: field "screenshot" is not a readable file: ${entry.screenshot}`);
|
||||
return { brief: entry.brief, screenshot: entry.screenshot };
|
||||
});
|
||||
}
|
||||
|
||||
export interface BatchVariant {
|
||||
variant: string;
|
||||
path: string;
|
||||
operation: "generate" | "evolve";
|
||||
status: "done" | "failed" | "rate_limited";
|
||||
error: string | null;
|
||||
retryable: boolean;
|
||||
check: { status: CheckResult["status"]; issues: string } | null;
|
||||
}
|
||||
|
||||
export interface BatchOptions {
|
||||
apiKey: string;
|
||||
outputDir: string;
|
||||
size: string;
|
||||
quality: string;
|
||||
fetchFn?: typeof globalThis.fetch;
|
||||
deadlineMs?: number;
|
||||
staggerMs?: number;
|
||||
}
|
||||
|
||||
const hasImage = (file: string) => fs.existsSync(file) && fs.statSync(file).size > 0;
|
||||
|
||||
/**
|
||||
* Generate one variant per brief with staggered parallel launches under one
|
||||
* batch deadline. Each variant repeats its own operation (generate, or evolve
|
||||
* when it has a screenshot) for one missing/empty output and for one failed
|
||||
* check, and always ends with a terminal status.
|
||||
*/
|
||||
export async function runBriefsBatch(entries: BriefEntry[], opts: BatchOptions): Promise<BatchVariant[]> {
|
||||
const fetchFn = opts.fetchFn ?? globalThis.fetch;
|
||||
const controller = new AbortController();
|
||||
const deadline = setTimeout(() => controller.abort(), opts.deadlineMs ?? BATCH_DEADLINE_MS);
|
||||
const signal = controller.signal;
|
||||
|
||||
const runOne = async (entry: BriefEntry, index: number): Promise<BatchVariant> => {
|
||||
const variant = String.fromCharCode(65 + index);
|
||||
const outputPath = path.join(opts.outputDir, `variant-${variant}.png`);
|
||||
const operation = entry.screenshot ? "evolve" : "generate";
|
||||
const finish = (status: BatchVariant["status"], error: string | null, retryable: boolean, check: BatchVariant["check"]): BatchVariant => {
|
||||
const detail = status === "done"
|
||||
? `${(fs.statSync(outputPath).size / 1024).toFixed(0)}KB, check ${check!.status}`
|
||||
: error;
|
||||
console.error(`VARIANT_${variant}_${status.toUpperCase()}: ${detail}`);
|
||||
return { variant, path: outputPath, operation, status, error, retryable, check };
|
||||
};
|
||||
const attempt = async (): Promise<VariantResult> => {
|
||||
if (!entry.screenshot) return generateVariant(opts.apiKey, entry.brief, outputPath, opts.size, opts.quality, fetchFn, signal);
|
||||
let analysis: string;
|
||||
try {
|
||||
analysis = await analyzeScreenshot(opts.apiKey, fs.readFileSync(entry.screenshot).toString("base64"), fetchFn, signal);
|
||||
} catch (err: any) {
|
||||
return { path: outputPath, success: false, error: signal.aborted ? DEADLINE_ERROR : `Screenshot analysis failed: ${err.message}`, retryable: true };
|
||||
}
|
||||
return generateVariant(opts.apiKey, evolvePrompt(analysis, entry.brief), outputPath, opts.size, opts.quality, fetchFn, signal);
|
||||
};
|
||||
const check = async (): Promise<NonNullable<BatchVariant["check"]>> => {
|
||||
if (signal.aborted) return { status: "skipped", issues: `${DEADLINE_ERROR} before the check` };
|
||||
try {
|
||||
const result = await checkMockup(outputPath, entry.brief, { apiKey: opts.apiKey, fetchFn, signal });
|
||||
return { status: result.status, issues: result.issues };
|
||||
} catch (err: any) {
|
||||
return { status: "skipped", issues: signal.aborted ? `${DEADLINE_ERROR} during the check` : `Check failed to run: ${err.message}` };
|
||||
}
|
||||
};
|
||||
|
||||
await sleep(index * (opts.staggerMs ?? STAGGER_MS), signal);
|
||||
if (signal.aborted) return finish("failed", `${DEADLINE_ERROR} before start`, true, null);
|
||||
console.error(` Starting variant ${variant} (${operation})...`);
|
||||
|
||||
let result = await attempt();
|
||||
if (result.success && !hasImage(outputPath)) result = await attempt();
|
||||
if (result.success && !hasImage(outputPath)) result = { path: outputPath, success: false, error: "Output file missing or empty", retryable: true };
|
||||
if (!result.success) return finish(result.rateLimited ? "rate_limited" : "failed", result.error ?? "unknown error", result.retryable ?? true, null);
|
||||
|
||||
let checked = await check();
|
||||
if (checked.status === "fail" && !signal.aborted) {
|
||||
const regenerated = await attempt();
|
||||
if (regenerated.success && hasImage(outputPath)) checked = await check();
|
||||
}
|
||||
return finish("done", null, false, checked);
|
||||
};
|
||||
|
||||
try {
|
||||
return await Promise.all(entries.map(runOne));
|
||||
} finally {
|
||||
clearTimeout(deadline);
|
||||
}
|
||||
}
|
||||
|
||||
/** Exit code for a finished batch: 0 all generated, 3 some failed, 1 none generated. */
|
||||
export function batchExitCode(results: BatchVariant[]): number {
|
||||
const done = results.filter(r => r.status === "done").length;
|
||||
if (done === results.length) return 0;
|
||||
return done === 0 ? 1 : 3;
|
||||
}
|
||||
|
||||
async function variantsFromBriefsFile(options: VariantsOptions): Promise<number> {
|
||||
if (typeof options.briefsFile !== "string") throw new Error("--briefs-file needs the path of a JSON file");
|
||||
if (options.brief !== undefined || options.briefFile !== undefined || options.viewports !== undefined) {
|
||||
throw new Error("--briefs-file cannot be combined with --brief, --brief-file or --viewports; put each variant's brief in the file.");
|
||||
}
|
||||
const entries = readBriefsFile(options.briefsFile);
|
||||
if (options.count !== undefined) console.error(`warning: --count is ignored with --briefs-file; generating ${entries.length} variants, one per entry.`);
|
||||
const apiKey = requireApiKey();
|
||||
fs.mkdirSync(options.outputDir, { recursive: true });
|
||||
|
||||
console.error(`Generating ${entries.length} variants from ${options.briefsFile}...`);
|
||||
const startTime = Date.now();
|
||||
const results = await runBriefsBatch(entries, {
|
||||
apiKey, outputDir: options.outputDir, size: options.size || "1536x1024", quality: options.quality || "high",
|
||||
});
|
||||
const done = results.filter(r => r.status === "done");
|
||||
console.error(`\n${done.length}/${results.length} variants generated (${((Date.now() - startTime) / 1000).toFixed(1)}s)`);
|
||||
console.log(JSON.stringify({
|
||||
outputDir: options.outputDir,
|
||||
count: results.length,
|
||||
succeeded: done.length,
|
||||
failed: results.length - done.length,
|
||||
validated: done.filter(r => r.check?.status === "pass").length,
|
||||
variants: results,
|
||||
}, null, 2));
|
||||
return batchExitCode(results);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate N variants with staggered parallel execution.
|
||||
* Generate N variants with staggered parallel execution. Returns the process
|
||||
* exit code (always 0 outside --briefs-file).
|
||||
*/
|
||||
export async function variants(options: VariantsOptions): Promise<void> {
|
||||
export async function variants(options: VariantsOptions): Promise<number> {
|
||||
if (options.briefsFile !== undefined) return variantsFromBriefsFile(options);
|
||||
const apiKey = requireApiKey();
|
||||
const baseBrief = options.briefFile
|
||||
? parseBrief(options.briefFile, true)
|
||||
@@ -160,7 +360,7 @@ export async function variants(options: VariantsOptions): Promise<void> {
|
||||
// If viewports specified, generate responsive variants instead of style variants
|
||||
if (options.viewports) {
|
||||
await generateResponsiveVariants(apiKey, baseBrief, options.outputDir, options.viewports, quality);
|
||||
return;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// #2032: normalize at the consumption site so every caller (CLI or
|
||||
@@ -229,6 +429,7 @@ export async function variants(options: VariantsOptions): Promise<void> {
|
||||
paths: succeeded,
|
||||
errors: failed,
|
||||
}, null, 2));
|
||||
return 0;
|
||||
}
|
||||
|
||||
const VIEWPORT_CONFIGS: Record<string, { size: string; suffix: string; desc: string }> = {
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
/**
|
||||
* Preloaded into the design CLI by variants-briefs.test.ts: replaces fetch so
|
||||
* `variants --briefs-file` runs end to end without network or billing.
|
||||
* BRIEFS_STUB_FAIL lists brief substrings whose image requests return 400.
|
||||
* Every call is appended to BRIEFS_STUB_LOG as one JSON line.
|
||||
*/
|
||||
import fs from "fs";
|
||||
|
||||
const TINY_PNG_BASE64 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkAAIAAAoAAv/lxKUAAAAASUVORK5CYII=";
|
||||
const failing = (process.env.BRIEFS_STUB_FAIL ?? "").split(",").filter(Boolean);
|
||||
|
||||
globalThis.fetch = (async (input: any, init?: any) => {
|
||||
const url = String(input);
|
||||
const body = String(init?.body ?? "");
|
||||
fs.appendFileSync(process.env.BRIEFS_STUB_LOG!, JSON.stringify({ url, body }) + "\n");
|
||||
if (url.endsWith("/v1/chat/completions")) {
|
||||
return Response.json({ choices: [{ message: { content: "PASS" } }] });
|
||||
}
|
||||
if (failing.some(token => body.includes(token))) return new Response("bad request", { status: 400 });
|
||||
return Response.json({ output: [{ type: "image_generation_call", result: TINY_PNG_BASE64 }] });
|
||||
}) as typeof globalThis.fetch;
|
||||
@@ -200,7 +200,7 @@ describe("image-generation call sites (Responses API)", () => {
|
||||
describe("vision call sites (Chat Completions)", () => {
|
||||
test("check", async () => {
|
||||
stubFetch((u) => ok(u, "PASS"));
|
||||
expect(await checkMockup(path.join(dir, "shot.png"), "brief")).toEqual({ pass: true, issues: "" });
|
||||
expect(await checkMockup(path.join(dir, "shot.png"), "brief")).toEqual({ pass: true, status: "pass", issues: "" });
|
||||
expectVisionShape(calls[0]);
|
||||
stubFetch(() => rejected());
|
||||
await checkMockup(path.join(dir, "shot.png"), "brief");
|
||||
@@ -256,7 +256,7 @@ describe("vision budgets leave room for reasoning", () => {
|
||||
|
||||
test("check, diff, memory, design-to-code and evolve analysis all answer", async () => {
|
||||
stubFetch(reasoningStub("PASS"));
|
||||
expect(await checkMockup(path.join(dir, "shot.png"), "brief")).toEqual({ pass: true, issues: "" });
|
||||
expect(await checkMockup(path.join(dir, "shot.png"), "brief")).toEqual({ pass: true, status: "pass", issues: "" });
|
||||
stubFetch(reasoningStub(JSON.stringify({ differences: [], summary: "same", matchScore: 100 })));
|
||||
expect((await diffMockups(path.join(dir, "shot.png"), path.join(dir, "shot.png"))).matchScore).toBe(100);
|
||||
stubFetch(reasoningStub(JSON.stringify({ colors: [], typography: [], spacing: [], layout: [], mood: "calm" })));
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
import { describe, test, expect, beforeEach, afterEach } from "bun:test";
|
||||
import { spawnSync } from "child_process";
|
||||
import fs from "fs";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import { batchExitCode, readBriefsFile, runBriefsBatch, type BriefEntry } from "../src/variants";
|
||||
|
||||
const TINY_PNG_BASE64 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkAAIAAAoAAv/lxKUAAAAASUVORK5CYII=";
|
||||
const CLI = path.resolve(import.meta.dir, "../src/cli.ts");
|
||||
const STUB = path.resolve(import.meta.dir, "fixtures/briefs-fetch-stub.ts");
|
||||
|
||||
type Call = { url: string; text: string };
|
||||
type Reply = (call: Call, nth: number) => Response;
|
||||
|
||||
const image = (result = TINY_PNG_BASE64) =>
|
||||
Response.json({ output: [{ type: "image_generation_call", result }] });
|
||||
const vision = (content: string) => Response.json({ choices: [{ message: { content } }] });
|
||||
|
||||
/** Routes image, check and screenshot-analysis requests to separate scripted replies. */
|
||||
function stubFetch(routes: { image?: Reply; check?: Reply; analysis?: Reply }) {
|
||||
const calls: Array<Call & { kind: string }> = [];
|
||||
const counts: Record<string, number> = { image: 0, check: 0, analysis: 0 };
|
||||
const fetchFn = (async (input: any, init?: any) => {
|
||||
const url = String(input);
|
||||
const text = String(init?.body ?? "");
|
||||
const kind = url.endsWith("/v1/responses") ? "image"
|
||||
: text.includes("UI quality checker") ? "check" : "analysis";
|
||||
const call = { url, text, kind };
|
||||
calls.push(call);
|
||||
const reply = routes[kind as keyof typeof routes];
|
||||
const nth = counts[kind]++;
|
||||
if (reply) return reply(call, nth);
|
||||
return kind === "image" ? image() : kind === "check" ? vision("PASS") : vision("A dashboard with a header.");
|
||||
}) as typeof globalThis.fetch;
|
||||
return { fetchFn, calls };
|
||||
}
|
||||
|
||||
let dir: string;
|
||||
let savedHome: string | undefined;
|
||||
|
||||
beforeEach(() => {
|
||||
dir = fs.mkdtempSync(path.join(os.tmpdir(), "variants-briefs-"));
|
||||
savedHome = process.env.GSTACK_HOME;
|
||||
process.env.GSTACK_HOME = dir;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (savedHome === undefined) delete process.env.GSTACK_HOME;
|
||||
else process.env.GSTACK_HOME = savedHome;
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const batch = (entries: BriefEntry[], fetchFn: typeof globalThis.fetch, extra: { deadlineMs?: number; staggerMs?: number } = {}) =>
|
||||
runBriefsBatch(entries, { apiKey: "sk-test", outputDir: dir, size: "1024x1024", quality: "low", fetchFn, staggerMs: 0, ...extra });
|
||||
|
||||
function writeBriefs(value: unknown): string {
|
||||
const file = path.join(dir, "briefs.json");
|
||||
fs.writeFileSync(file, JSON.stringify(value));
|
||||
return file;
|
||||
}
|
||||
|
||||
describe("readBriefsFile", () => {
|
||||
test("accepts 1 to 7 entries with optional screenshots", () => {
|
||||
const shot = path.join(dir, "current.png");
|
||||
fs.writeFileSync(shot, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
expect(readBriefsFile(writeBriefs([{ brief: "a" }, { brief: "b", screenshot: shot }])))
|
||||
.toEqual([{ brief: "a" }, { brief: "b", screenshot: shot }]);
|
||||
});
|
||||
|
||||
test.each([
|
||||
["not an array", { brief: "a" }, "JSON array of 1 to 7"],
|
||||
["empty", [], "JSON array of 1 to 7"],
|
||||
["too many", Array.from({ length: 8 }, () => ({ brief: "x" })), "JSON array of 1 to 7"],
|
||||
["missing brief", [{ brief: "ok" }, { screenshot: "x.png" }], 'entry 1: field "brief"'],
|
||||
["blank brief", [{ brief: " " }], 'entry 0: field "brief"'],
|
||||
["unknown field", [{ brief: "a", count: 3 }], 'entry 0: unknown field "count"'],
|
||||
["unreadable screenshot", [{ brief: "a", screenshot: "/nonexistent/shot.png" }], 'entry 0: field "screenshot" is not a readable file'],
|
||||
])("%s names the file, entry and field", (_label, value, message) => {
|
||||
const file = writeBriefs(value);
|
||||
expect(() => readBriefsFile(file)).toThrow(message);
|
||||
expect(() => readBriefsFile(file)).toThrow(file);
|
||||
});
|
||||
|
||||
test("malformed JSON names the file", () => {
|
||||
const file = path.join(dir, "bad.json");
|
||||
fs.writeFileSync(file, "[{");
|
||||
expect(() => readBriefsFile(file)).toThrow(`--briefs-file ${file}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("runBriefsBatch", () => {
|
||||
test("each variant sends its own brief and ends DONE with a passing check", async () => {
|
||||
const { fetchFn, calls } = stubFetch({});
|
||||
const results = await batch([{ brief: "CALM-DASHBOARD" }, { brief: "BOLD-DASHBOARD" }], fetchFn);
|
||||
expect(results.map(r => [r.variant, r.status, r.check?.status, r.operation])).toEqual([
|
||||
["A", "done", "pass", "generate"], ["B", "done", "pass", "generate"],
|
||||
]);
|
||||
const images = calls.filter(c => c.kind === "image");
|
||||
expect(images).toHaveLength(2);
|
||||
expect(images.filter(c => c.text.includes("CALM-DASHBOARD"))).toHaveLength(1);
|
||||
expect(images.filter(c => c.text.includes("BOLD-DASHBOARD"))).toHaveLength(1);
|
||||
expect(fs.statSync(path.join(dir, "variant-B.png")).size).toBeGreaterThan(0);
|
||||
expect(batchExitCode(results)).toBe(0);
|
||||
});
|
||||
|
||||
test("429 backs off and retries the same request", async () => {
|
||||
const { fetchFn, calls } = stubFetch({
|
||||
image: (_call, nth) => nth === 0 ? new Response("slow down", { status: 429, headers: { "Retry-After": "0" } }) : image(),
|
||||
});
|
||||
const [result] = await batch([{ brief: "one" }], fetchFn);
|
||||
expect(result!.status).toBe("done");
|
||||
expect(calls.filter(c => c.kind === "image")).toHaveLength(2);
|
||||
});
|
||||
|
||||
test("exhausted 429 retries end RATE_LIMITED and retryable", async () => {
|
||||
const { fetchFn } = stubFetch({ image: () => new Response("slow down", { status: 429, headers: { "Retry-After": "0" } }) });
|
||||
const [result] = await batch([{ brief: "one" }], fetchFn);
|
||||
expect(result).toMatchObject({ status: "rate_limited", retryable: true, check: null });
|
||||
expect(batchExitCode([result!])).toBe(1);
|
||||
});
|
||||
|
||||
test("an empty output file is regenerated once", async () => {
|
||||
const { fetchFn, calls } = stubFetch({ image: (_call, nth) => nth === 0 ? image("=") : image() });
|
||||
const [result] = await batch([{ brief: "one" }], fetchFn);
|
||||
expect(result!.status).toBe("done");
|
||||
expect(calls.filter(c => c.kind === "image")).toHaveLength(2);
|
||||
});
|
||||
|
||||
test("a failed check regenerates once with the same brief, then rechecks", async () => {
|
||||
const { fetchFn, calls } = stubFetch({ check: (_call, nth) => vision(nth === 0 ? "FAIL: header text is garbled" : "PASS") });
|
||||
const [result] = await batch([{ brief: "CHECK-ME" }], fetchFn);
|
||||
expect(result).toMatchObject({ status: "done", check: { status: "pass", issues: "" } });
|
||||
const images = calls.filter(c => c.kind === "image");
|
||||
expect(images).toHaveLength(2);
|
||||
expect(images.every(c => c.text.includes("CHECK-ME"))).toBe(true);
|
||||
expect(calls.filter(c => c.kind === "check")).toHaveLength(2);
|
||||
});
|
||||
|
||||
test("an unavailable check is reported as skipped, never as validated", async () => {
|
||||
const { fetchFn } = stubFetch({ check: () => new Response("down", { status: 503 }) });
|
||||
const [result] = await batch([{ brief: "one" }], fetchFn);
|
||||
expect(result).toMatchObject({ status: "done", check: { status: "skipped" } });
|
||||
});
|
||||
|
||||
test("screenshot entries evolve, and a failed check repeats the evolve operation", async () => {
|
||||
const shot = path.join(dir, "current.png");
|
||||
fs.writeFileSync(shot, Buffer.from(TINY_PNG_BASE64, "base64"));
|
||||
const { fetchFn, calls } = stubFetch({ check: (_call, nth) => vision(nth === 0 ? "FAIL: cramped" : "PASS") });
|
||||
const [result] = await batch([{ brief: "EVOLVE-ME", screenshot: shot }], fetchFn);
|
||||
expect(result).toMatchObject({ operation: "evolve", status: "done", check: { status: "pass" } });
|
||||
expect(calls.map(c => c.kind)).toEqual(["analysis", "image", "check", "analysis", "image", "check"]);
|
||||
expect(calls.filter(c => c.kind === "image").every(c => c.text.includes("REQUESTED CHANGES") && c.text.includes("EVOLVE-ME"))).toBe(true);
|
||||
});
|
||||
|
||||
test("a non-retryable API error fails only its own variant", async () => {
|
||||
const { fetchFn } = stubFetch({ image: call => call.text.includes("BROKEN") ? new Response("bad", { status: 400 }) : image() });
|
||||
const results = await batch([{ brief: "fine" }, { brief: "BROKEN" }], fetchFn);
|
||||
expect(results.map(r => [r.status, r.retryable])).toEqual([["done", false], ["failed", false]]);
|
||||
expect(batchExitCode(results)).toBe(3);
|
||||
});
|
||||
|
||||
test("the batch deadline stops new launches and in-flight requests; every variant is terminal", async () => {
|
||||
const { fetchFn, calls } = stubFetch({});
|
||||
const slow = (async (input: any, init?: any) => {
|
||||
if (String(input).endsWith("/v1/responses") && String(init?.body).includes("SLOW")) {
|
||||
return new Promise<Response>((_resolve, reject) => {
|
||||
init.signal.addEventListener("abort", () => reject(Object.assign(new Error("aborted"), { name: "AbortError" })));
|
||||
});
|
||||
}
|
||||
return fetchFn(input, init);
|
||||
}) as typeof globalThis.fetch;
|
||||
const started = Date.now();
|
||||
const results = await batch([{ brief: "SLOW" }, { brief: "late" }], slow, { deadlineMs: 300, staggerMs: 5_000 });
|
||||
expect(Date.now() - started).toBeLessThan(3_000);
|
||||
expect(results.map(r => [r.status, r.retryable, r.error])).toEqual([
|
||||
["failed", true, "Batch deadline reached"],
|
||||
["failed", true, "Batch deadline reached before start"],
|
||||
]);
|
||||
expect(calls).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("$D variants --briefs-file (CLI end to end with a stubbed fetch)", () => {
|
||||
const run = (args: string[], fail = "") => {
|
||||
const log = path.join(dir, "fetch.log");
|
||||
fs.writeFileSync(log, "");
|
||||
const result = spawnSync(process.execPath, ["--preload", STUB, CLI, "variants", ...args], {
|
||||
cwd: dir,
|
||||
env: { PATH: process.env.PATH!, HOME: dir, GSTACK_HOME: dir, OPENAI_API_KEY: "sk-test-stub", BRIEFS_STUB_LOG: log, BRIEFS_STUB_FAIL: fail },
|
||||
encoding: "utf-8",
|
||||
timeout: 60_000,
|
||||
});
|
||||
const calls = fs.readFileSync(log, "utf-8").split("\n").filter(Boolean);
|
||||
return { ...result, calls };
|
||||
};
|
||||
const out = () => path.join(dir, "out");
|
||||
|
||||
test("all variants succeed: exit 0, JSON on stdout, status lines on stderr", () => {
|
||||
const file = writeBriefs([{ brief: "FIRST" }, { brief: "SECOND" }]);
|
||||
const r = run(["--briefs-file", file, "--output-dir", out(), "--count", "5"]);
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
const json = JSON.parse(r.stdout);
|
||||
expect(json).toMatchObject({ count: 2, succeeded: 2, failed: 0, validated: 2 });
|
||||
expect(json.variants.map((v: any) => [v.variant, v.status, v.error, v.retryable])).toEqual([["A", "done", null, false], ["B", "done", null, false]]);
|
||||
expect(r.stderr).toContain("--count is ignored");
|
||||
expect(r.stderr).toMatch(/VARIANT_A_DONE: /);
|
||||
expect(r.stderr).toMatch(/VARIANT_B_DONE: /);
|
||||
expect(fs.existsSync(path.join(out(), "variant-B.png"))).toBe(true);
|
||||
});
|
||||
|
||||
test("some variants fail: exit 3", () => {
|
||||
const file = writeBriefs([{ brief: "FIRST" }, { brief: "BROKEN" }]);
|
||||
const r = run(["--briefs-file", file, "--output-dir", out()], "BROKEN");
|
||||
expect(r.status, r.stderr).toBe(3);
|
||||
expect(JSON.parse(r.stdout).variants[1]).toMatchObject({ status: "failed", retryable: false });
|
||||
expect(r.stderr).toMatch(/VARIANT_B_FAILED: API error \(400\)/);
|
||||
});
|
||||
|
||||
test("all variants fail: exit 1", () => {
|
||||
const file = writeBriefs([{ brief: "BROKEN" }]);
|
||||
expect(run(["--briefs-file", file, "--output-dir", out()], "BROKEN").status).toBe(1);
|
||||
});
|
||||
|
||||
test("invalid input exits 1 before any billable call", () => {
|
||||
const file = writeBriefs([{ brief: "fine" }, { brief: "" }]);
|
||||
const invalid = run(["--briefs-file", file, "--output-dir", out()]);
|
||||
expect(invalid.status).toBe(1);
|
||||
expect(invalid.stderr).toContain('entry 1: field "brief"');
|
||||
expect(invalid.calls).toHaveLength(0);
|
||||
const conflict = run(["--briefs-file", writeBriefs([{ brief: "a" }]), "--brief", "also", "--output-dir", out()]);
|
||||
expect(conflict.status).toBe(1);
|
||||
expect(conflict.stderr).toContain("cannot be combined with --brief");
|
||||
expect(conflict.calls).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -81,6 +81,7 @@ export const CASE_TEST_NAMES: Record<string, string> = {
|
||||
'plan-review-report': '/plan-eng-review writes GSTACK REVIEW REPORT to plan file',
|
||||
'auq-format-gate': "/plan-ceo-review's first AskUserQuestion is a compliant decision brief (7/7 + substance)",
|
||||
'autoplan-dual-voice': 'both Claude + Codex voices produce output in Phase 1 (within timeout)',
|
||||
'plan-ceo-review-plan-mode': 'first terminal outcome is asked (Step 0 fires before any plan write)',
|
||||
};
|
||||
|
||||
export const CASE_KEY_SEPARATOR = '#';
|
||||
|
||||
@@ -148,6 +148,7 @@
|
||||
"test/skill-e2e-office-hours.test.ts": 136970,
|
||||
"test/skill-e2e-outside-plan-disabled.test.ts": 53292,
|
||||
"test/skill-e2e-outside-voice.test.ts": 1000,
|
||||
"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts": 56080,
|
||||
"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts": 85282,
|
||||
"test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash.test.ts": 80465,
|
||||
"test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts": 49282,
|
||||
|
||||
@@ -1,47 +1,26 @@
|
||||
/**
|
||||
* {{DESIGN_DOC_DISCOVERY}} — the canonical design-doc discovery block (#703).
|
||||
* {{DESIGN_DOC_DISCOVERY}} — the canonical design-doc discovery block.
|
||||
*
|
||||
* Finds the design doc a plan review should read: newest branch-scoped doc
|
||||
* under ~/.gstack/projects/<slug>/, falling back to newest project-scoped
|
||||
* doc, then lets a repo-local doc (DESIGN.md or docs/designs/*.md) win when
|
||||
* it is at least as fresh. office-hours dual-writes docs/designs/ alongside
|
||||
* ~/.gstack, and the committed copy is what teammates see — but a stale old
|
||||
* repo doc must never shadow a newer private session.
|
||||
*
|
||||
* Single source of truth for the freshness-preference logic that previously
|
||||
* lived verbatim in plan-ceo-review, plan-eng-review, plan-devex-review, and
|
||||
* the prerequisite-skill re-check in review.ts (GStack 2 fork-port wave,
|
||||
* time-attack/gstack). Drift between copies meant plan reviews could
|
||||
* disagree about which design doc wins.
|
||||
* One invocation of bin/gstack-design-doc-find, which owns the precedence
|
||||
* (newest branch-scoped doc in the state root's project directory, then the
|
||||
* newest project-scoped doc, then a repo-local DESIGN.md or docs/designs/*.md
|
||||
* when it is at least as fresh) and resolves the state root through
|
||||
* bin/gstack-paths. Plan reviews, autoplan and the prerequisite-skill re-check
|
||||
* in spec-review.ts all render this one block, so they agree on which doc wins.
|
||||
*
|
||||
* The fragment carries no code fences — the {{DESIGN_DOC_DISCOVERY}} token
|
||||
* sits inside each caller's ```bash block. Callers must set $SLUG and
|
||||
* $BRANCH first (and `setopt +o nomatch` for zsh); the block sets $DESIGN
|
||||
* and prints "Design doc found: ..." or "No design doc found".
|
||||
* $BRANCH first; the block sets $DESIGN and prints "Design doc found: ..." or
|
||||
* "No design doc found".
|
||||
*/
|
||||
|
||||
import type { TemplateContext } from './types';
|
||||
|
||||
/**
|
||||
* Raw canonical fragment, exported so TS resolvers (review.ts's prerequisite
|
||||
* re-check) can interpolate it into their own template strings instead of
|
||||
* embedding a drifting copy.
|
||||
* Raw canonical fragment, exported so TS resolvers (spec-review.ts's
|
||||
* prerequisite re-check) can interpolate it into their own template strings.
|
||||
*/
|
||||
export const DESIGN_DOC_DISCOVERY_BLOCK = `_LOCALDOC=$(ls -t ~/.gstack/projects/$SLUG/*-$BRANCH-design-*.md 2>/dev/null | head -1)
|
||||
[ -z "$_LOCALDOC" ] && _LOCALDOC=$(ls -t ~/.gstack/projects/$SLUG/*-design-*.md 2>/dev/null | head -1)
|
||||
# Repo-local docs win when at least as fresh (#703): office-hours dual-writes
|
||||
# docs/designs/ alongside ~/.gstack, and the committed copy is what teammates
|
||||
# see. A stale old repo doc never shadows a newer private session.
|
||||
_REPOTOP=$(git rev-parse --show-toplevel 2>/dev/null || echo "")
|
||||
_REPODOC=""
|
||||
if [ -n "$_REPOTOP" ]; then
|
||||
[ -f "$_REPOTOP/DESIGN.md" ] && _REPODOC="$_REPOTOP/DESIGN.md"
|
||||
[ -z "$_REPODOC" ] && _REPODOC=$(ls -t "$_REPOTOP"/docs/designs/*.md 2>/dev/null | head -1)
|
||||
fi
|
||||
DESIGN="$_LOCALDOC"
|
||||
if [ -n "$_REPODOC" ] && { [ -z "$_LOCALDOC" ] || [ "$_REPODOC" -nt "$_LOCALDOC" ]; }; then
|
||||
DESIGN="$_REPODOC"
|
||||
fi
|
||||
export const DESIGN_DOC_DISCOVERY_BLOCK = `DESIGN=$(~/.claude/skills/gstack/bin/gstack-design-doc-find "$SLUG" "$BRANCH")
|
||||
[ -n "$DESIGN" ] && echo "Design doc found: $DESIGN" || echo "No design doc found"`;
|
||||
|
||||
export function generateDesignDocDiscovery(_ctx: TemplateContext): string {
|
||||
|
||||
+18
-15
@@ -76,30 +76,33 @@ For /qa and /qa-only:
|
||||
- Browser Quick: SECONDS=30. Browser Full/Regression: SECONDS=900.
|
||||
- Functional Full, Quick and Regression have no default total timer.
|
||||
Set SECONDS to the shorter mode/caller limit; an unlimited mode uses the caller's bound.
|
||||
Without a total time limit, do not start D; announce finite command timeouts.
|
||||
Without a total time limit, do not create DEADLINE_FILE; announce finite command timeouts.
|
||||
Stop when scoped contracts are tested or blocked.
|
||||
Clocks/checkpoints use REPORT_DIR; mixed standalone runs use REPORT_DIR/browser and REPORT_DIR/functional, with one final report at REPORT_DIR. Caller paths win.
|
||||
Names used below (quote every path):
|
||||
- R: this surface's owned probe directory, chosen above. D: \`R/deadline.json\`, which exists only for a bounded run.
|
||||
- PROBE_DIR: this surface's owned probe directory, chosen above. DEADLINE_FILE: \`PROBE_DIR/deadline.json\`, which exists only for a bounded run.
|
||||
- SECONDS: the total probe budget set above. NNN: a fresh three-digit ID (001, 002, ...) for each capture or checkpoint; never reuse one.
|
||||
- G = \`${quoteSafePath(ctx.paths.binDir)}/gstack-qa-deadline\`, the deadline guard: \`start\` creates D, \`status\` prints \`remainingMs\` and \`expired\`, and \`run\` executes one command, stopping it at the deadline.
|
||||
- Q = \`${quoteSafePath(ctx.paths.binDir)}/gstack-qa-evidence\`, the functional evidence recorder: \`capture\` runs one command, stores its exit code, stdout and stderr under \`R/.qa-evidence/NNN/\`, and prints status \`complete\`, \`incomplete\` or \`sensitive\`.
|
||||
- DEADLINE_TOOL = \`${quoteSafePath(ctx.paths.binDir)}/gstack-qa-deadline\`, the deadline guard: \`start\` creates DEADLINE_FILE, \`status\` prints \`remainingMs\` and \`expired\`, and \`run\` executes one command, stopping it at the deadline.
|
||||
- EVIDENCE_TOOL = \`${quoteSafePath(ctx.paths.binDir)}/gstack-qa-evidence\`, the functional evidence recorder: \`capture\` runs one command, stores its exit code, stdout and stderr under \`PROBE_DIR/.qa-evidence/NNN/\`, and prints status \`complete\`, \`incomplete\` or \`sensitive\`.
|
||||
|
||||
Start once before baseline: \`bun G start D SECONDS [EARLIER_UTC]\` if bounded.
|
||||
Start once before baseline: \`bun DEADLINE_TOOL start DEADLINE_FILE SECONDS [EARLIER_UTC]\` if bounded.
|
||||
EARLIER_UTC = caller's absolute deadline, if set.
|
||||
Functional: \`bun Q capture R NNN [--public] --deadline D -- COMMAND ARGS\`.
|
||||
Unbounded: use \`--timeout-ms MS\` instead of \`--deadline D\`.
|
||||
--public requires approved public/synthetic output; Q screens credentials. For complete private captures, await a safe Read of \`R/.qa-evidence/NNN/observation.json\` (the decoded stdout). Sensitive/incomplete captures cannot anchor checkpoints.
|
||||
Bounded browsers: \`bun G run D -- COMMAND ARGS\`. No detached probes.
|
||||
Never reset D/bypass G. Expiry or invalid/missing D stops probes; report unfinished coverage. QA_DEADLINE receipts are not observations.
|
||||
Functional: \`bun EVIDENCE_TOOL capture PROBE_DIR NNN [--public] --deadline DEADLINE_FILE -- COMMAND ARGS\`.
|
||||
Unbounded: use \`--timeout-ms MS\` instead of \`--deadline DEADLINE_FILE\`.
|
||||
--public requires approved public/synthetic output; EVIDENCE_TOOL screens credentials. For complete private captures, await a safe Read of \`PROBE_DIR/.qa-evidence/NNN/observation.json\` (the decoded stdout). Sensitive/incomplete captures cannot anchor checkpoints.
|
||||
Bounded browsers: \`bun DEADLINE_TOOL run DEADLINE_FILE -- COMMAND ARGS\`. No detached probes.
|
||||
Never reset DEADLINE_FILE/bypass DEADLINE_TOOL. Expiry or an invalid/missing DEADLINE_FILE stops probes; report unfinished coverage. QA_DEADLINE receipts are not observations.
|
||||
|
||||
## 2. Probe loop
|
||||
|
||||
Each probe is one native command/interaction plus checks, excluding bookkeeping.
|
||||
Never batch probes.
|
||||
How one probe works: (1) if bounded, check the clock and choose the next command;
|
||||
(2) publish a checkpoint naming it (functional: \`capture --after\` publishes it);
|
||||
(3) run exactly that command via DEADLINE_TOOL or EVIDENCE_TOOL, read its result, repeat.
|
||||
|
||||
1. First demonstrate success: output AND durable effects. Guard if bounded; await completion.
|
||||
2. **Decide whether another probe is needed.** If bounded, run \`bun G status D\`.
|
||||
2. **Decide whether another probe is needed.** If bounded, run \`bun DEADLINE_TOOL status DEADLINE_FILE\`.
|
||||
If expired or no safe next probe remains, STOP exploration; write the report, not a checkpoint.
|
||||
${reportOnly ? ` **Classify the last result before copying it.** For public or synthetic observations,
|
||||
retain the entire result unchanged, including owned fixture paths, IDs, hashes and
|
||||
@@ -119,11 +122,11 @@ ${reportOnly ? ` For guarded text, copy the complete span between the guard's
|
||||
Preserve every safe program-JSON key/value and identity hash unchanged.
|
||||
Withhold unsafe values, disclose limits and stop that chain.
|
||||
Check fields before publication. No drafts/placeholders or invented safe-path redactions; corrections cannot repair published notes.
|
||||
Functional: do not write this file; the next capture publishes it: \`bun Q capture R NNN --deadline D --after PREV --hypothesis 'why' -- CMD\` (PREV: the last complete capture's ID). Q supplies observed; never transcribe it.
|
||||
Functional: do not write this file; the next capture publishes it: \`bun EVIDENCE_TOOL capture PROBE_DIR NNN --deadline DEADLINE_FILE --after PREV --hypothesis 'why' -- CMD\` (PREV: the last complete capture's ID). EVIDENCE_TOOL supplies observed; never transcribe it.
|
||||
Browser checkpoints use Write: create the four-field file yourself.
|
||||
Wait for successful checkpoint publication before dispatch.
|
||||
Never backfill or overwrite notes.
|
||||
3. Run that exact probe; G enforces the deadline when bounded.
|
||||
3. Run that exact probe; DEADLINE_TOOL enforces the deadline when bounded.
|
||||
Report refusals as not-run; retain initial state/inputs/results. Repeat from step 2.
|
||||
4. Replay the exact failing command/request from the same initial fixture state via steps 2–3 (same native command, fresh capture ID)
|
||||
${reportOnly ? 'to confirm it' : 'before repair'}, then minimize via those gates. Expiry leaves confirmation/minimization incomplete.
|
||||
@@ -147,7 +150,7 @@ Never freeze buggy output, weaken tests or delete valid red tests.`}
|
||||
## 4. Final report
|
||||
|
||||
Use the surface report template; link each checkpoint. Separate browser scores, functional outcomes and proposed/executed tests.
|
||||
Write R/annotations.json {evidence: [{capture, command, contract, expected, classification}], limits} (browser-only: evidence [], checkpoints in limits); before Markdown \`bun Q materialize R annotations.json\` (fills observed/metadata; prints reportLinks); you classify. Retain all safe probes, including failures/replays; disclose withheld/incomplete evidence.
|
||||
Write PROBE_DIR/annotations.json {evidence: [{capture, command, contract, expected, classification}], limits} (browser-only: evidence [], checkpoints in limits); before Markdown \`bun EVIDENCE_TOOL materialize PROBE_DIR annotations.json\` (fills observed/metadata; prints reportLinks); you classify. Retain all safe probes, including failures/replays; disclose withheld/incomplete evidence.
|
||||
Evidence is invocation-local${reportOnly ? '.' : '; /ship reruns once per invocation.'}
|
||||
Missing prerequisites/expectations/observations, timeouts and refusal never pass.
|
||||
Pass requires all required current-input contracts to pass with no required remainder.
|
||||
@@ -260,7 +263,7 @@ Never install, import cookies or bootstrap tests. Functional-only skips browser
|
||||
|
||||
**3. Run smoke and plan checks.**
|
||||
Follow the shared Probe loop for smoke checks and replays until the smoke limit.
|
||||
Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard; never reset the clock. Their checkpoints sit beside D; they skip \`G status D\` and use \`--timeout-ms\`, not \`--deadline D\`. Post-expiry smoke rechecks are not-run.
|
||||
Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard; never reset the clock. Their checkpoints sit beside DEADLINE_FILE; they skip \`DEADLINE_TOOL status DEADLINE_FILE\` and use \`--timeout-ms\`, not \`--deadline DEADLINE_FILE\`. Post-expiry smoke rechecks are not-run.
|
||||
Use finite command timeouts, capped at the caller's remaining time if it has a deadline.${ship ? '' : ' /review sets none; only an invoker-supplied EARLIER_UTC counts.'}
|
||||
Await clock/guard results before acting. When the caller's deadline expires, mark unfinished checks not-run.
|
||||
|
||||
|
||||
@@ -26,12 +26,13 @@ export const PR_PROFILE_CASE_IDS = [
|
||||
'gstack-upgrade-happy-path',
|
||||
'investigate-owned-completion', 'investigate-owned-abort', 'investigate-owned-ending-error',
|
||||
'ship-coverage-value', 'review-test-value', 'test-audit-report-only',
|
||||
'office-hours-auto-mode',
|
||||
'office-hours-auto-mode', 'plan-ceo-review-plan-mode',
|
||||
] as const;
|
||||
|
||||
/** Audited ownership: unknown/direct-describe files remain broad coverage. */
|
||||
export const PR_PROFILE_FILES: Record<string, readonly string[]> = {
|
||||
'test/skill-e2e-office-hours-auto-mode.test.ts': ['office-hours-auto-mode'],
|
||||
'test/skill-e2e-plan-ceo-plan-mode.test.ts': ['plan-ceo-review-plan-mode'],
|
||||
'test/skill-e2e-investigate-owned-completion.test.ts': ['investigate-owned-completion'],
|
||||
'test/skill-e2e-investigate-owned-termination.test.ts': ['investigate-owned-abort', 'investigate-owned-ending-error'],
|
||||
'test/skill-e2e-hermetic-canary.test.ts': ['hermetic-canary', 'hermetic-sentinel'],
|
||||
|
||||
@@ -23,7 +23,7 @@ function fixture(plan = original) {
|
||||
const env = { PATH: process.env.PATH!, HOME: home, GSTACK_HOME: state, GSTACK_STATE_ROOT: state };
|
||||
const start = () => execFileSync(join(root, 'bin/gstack-skill-start'), ['--skill', 'autoplan'],
|
||||
{ cwd, env, encoding: 'utf8', timeout: 30_000 });
|
||||
const discover = () => execFileSync('bash', ['-c', DESIGN_DOC_DISCOVERY_BLOCK],
|
||||
const discover = () => execFileSync('bash', ['-c', DESIGN_DOC_DISCOVERY_BLOCK.replaceAll('~/.claude/skills/gstack/bin/', `${root}/bin/`)],
|
||||
{ cwd, env: { ...env, SLUG: 'chain-fixture', BRANCH: 'main' }, encoding: 'utf8', timeout: 5000 }).trim();
|
||||
return { cwd, home, state, planFile, start, discover, cleanup: () => rmSync(temp, { recursive: true, force: true }) };
|
||||
}
|
||||
|
||||
@@ -301,7 +301,8 @@ describe('CEO finding fixture establishes scope before launch', () => {
|
||||
const plan = '# Export saved settings\nReview the CSV formatter before implementation.\n';
|
||||
const design = '# Settings export design\n\n## Problem\nOperators need saved settings in a spreadsheet for offline comparison.\n\n## Approach\nReuse the settings API and escape commas, quotes, and newlines in a CSV formatter.\n';
|
||||
seedCeoFindingProject(cwd, plan, design);
|
||||
const output = execFileSync('bash', ['-c', `SLUG=fixture; BRANCH=main; ${DESIGN_DOC_DISCOVERY_BLOCK}`], {
|
||||
const block = DESIGN_DOC_DISCOVERY_BLOCK.replaceAll('~/.claude/skills/gstack/bin/', `${path.resolve(import.meta.dir, '..')}/bin/`);
|
||||
const output = execFileSync('bash', ['-c', `SLUG=fixture; BRANCH=main; ${block}`], {
|
||||
cwd, env: { PATH: process.env.PATH!, HOME: home }, encoding: 'utf8', timeout: 10_000,
|
||||
});
|
||||
expect(output).toBe(`Design doc found: ${path.join(cwd, 'DESIGN.md')}\n`);
|
||||
|
||||
@@ -80,10 +80,10 @@ test('actual CLI quality check distinguishes failure from skipped coverage despi
|
||||
try {
|
||||
const failed = f.run(['check', '--image', f.image, '--brief', 'Readable title'], { MODE: 'check' });
|
||||
expect(failed.status, failed.stderr).toBe(0);
|
||||
expect(JSON.parse(failed.stdout)).toEqual({ pass: false, issues: 'illegible title' });
|
||||
expect(JSON.parse(failed.stdout)).toEqual({ pass: false, status: 'fail', issues: 'illegible title' });
|
||||
const unavailable = f.run(['check', '--image', f.image, '--brief', 'Readable title'], { MODE: 'unavailable' });
|
||||
expect(unavailable.status, unavailable.stderr).toBe(0);
|
||||
expect(JSON.parse(unavailable.stdout)).toEqual({ pass: true, issues: 'Vision check unavailable — skipped' });
|
||||
expect(JSON.parse(unavailable.stdout)).toEqual({ pass: true, status: 'skipped', issues: 'Vision check unavailable — skipped' });
|
||||
} finally { f.cleanup(); }
|
||||
});
|
||||
|
||||
|
||||
@@ -133,13 +133,13 @@ test('live periodic census fits the declared CI wall including setup', () => {
|
||||
const plannedFiles = new Set(m.entries.filter(e => e.status === 'planned').map(e => shardFile(e.file)));
|
||||
expect(plannedFiles).toEqual(new Set(selectPaidTestFiles(collectPaidTestFiles(), 'periodic').selected));
|
||||
const overlays = m.entries.filter(e => e.status === 'planned' && e.slice === m.sliceCount);
|
||||
expect(overlays).toHaveLength(4);
|
||||
expect(overlays).toHaveLength(5);
|
||||
expect(overlays.every(e => isOverlayTestFile(e.file))).toBe(true);
|
||||
});
|
||||
|
||||
test('registered allocation is deterministic and preserves every discovered file', () => {
|
||||
const files = collectPaidTestFiles();
|
||||
expect(files).toHaveLength(106);
|
||||
expect(files).toHaveLength(107);
|
||||
expect(files).toContain('test/skill-e2e-ship-skip.test.ts');
|
||||
const m = livePlan(files);
|
||||
expect(livePlan([...files].reverse())).toEqual(m);
|
||||
@@ -189,10 +189,10 @@ test('current detach supervision covers the live-census floor', () => {
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(import.meta.dir, '../package.json'), 'utf8'));
|
||||
const periodicTimeout = Number(pkg.scripts['eval:bg:periodic'].match(/--timeout\s+(\d+)/)[1]);
|
||||
const gateTimeout = Number(pkg.scripts['eval:bg:gate'].match(/--timeout\s+(\d+)/)[1]);
|
||||
expect(floorFor('gate')).toBe(21_725);
|
||||
expect(floorFor('gate')).toBe(22_355);
|
||||
expect(gateTimeout).toBe(49_320);
|
||||
expect(gateTimeout).toBeGreaterThanOrEqual(floorFor('gate'));
|
||||
expect(floorFor('periodic')).toBe(35_711);
|
||||
expect(floorFor('periodic')).toBe(35_742);
|
||||
expect(periodicTimeout).toBeGreaterThanOrEqual(floorFor('periodic'));
|
||||
});
|
||||
|
||||
|
||||
Vendored
+31
@@ -325,6 +325,37 @@ export const OVERLAY_FIXTURES: OverlayFixture[] = [
|
||||
pass: lowerIsBetter20Pct,
|
||||
},
|
||||
|
||||
// Opus 5.5 measures the generic claude overlay (resolveModel maps
|
||||
// claude-opus-5-5 to `claude`); it adds no Opus 5.5 nudges.
|
||||
{
|
||||
id: 'claude-dedicated-tools-vs-bash-opus-5-5',
|
||||
overlayPath: 'model-overlays/claude.md',
|
||||
model: 'claude-opus-5-5',
|
||||
trials: 10,
|
||||
concurrency: 3,
|
||||
direction: 'lower_is_better',
|
||||
maxTurns: 15,
|
||||
setupWorkspace: (dir) => {
|
||||
fs.mkdirSync(path.join(dir, 'src'), { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'src', 'index.ts'), "export const x = 1;\n");
|
||||
fs.writeFileSync(path.join(dir, 'src', 'util.ts'), "export function util() { return 42; }\n");
|
||||
fs.writeFileSync(path.join(dir, 'src', 'types.ts'), "export type Foo = { a: number };\n");
|
||||
fs.writeFileSync(path.join(dir, 'src', 'config.ts'), "export const c = { n: 'demo' };\n");
|
||||
fs.writeFileSync(path.join(dir, 'src', 'api.ts'), "export async function fetchFoo() { return null; }\n");
|
||||
},
|
||||
userPrompt:
|
||||
"List every TypeScript file under src/ and tell me what each exports. " +
|
||||
"Return only a JSON object mapping each src/ path to an array of its exported symbol names. " +
|
||||
"The final message is consumed directly by JSON.parse: return the exact JSON object, with no Markdown fences and no other prose. " +
|
||||
"You may use any tools available.",
|
||||
metric: bashToolCallCount,
|
||||
metricName: 'bash_tool_calls',
|
||||
verify: (r) => assertFinalJson(r, { 'src/index.ts': ['x'], 'src/util.ts': ['util'], 'src/types.ts': ['Foo'], 'src/config.ts': ['c'], 'src/api.ts': ['fetchFoo'] }),
|
||||
gate: lowerIsBetter20PctOrZeroBaseline,
|
||||
comparison: { direction: 'lower_is_better', minimum: 0 },
|
||||
pass: lowerIsBetter20Pct,
|
||||
},
|
||||
|
||||
];
|
||||
|
||||
// Validate at module load so a broken fixture fails fast at test startup,
|
||||
|
||||
@@ -1927,6 +1927,10 @@ describe('BENEFITS_FROM resolver', () => {
|
||||
fs.chmodSync(helper, 0o755);
|
||||
fs.copyFileSync(path.join(ROOT, 'bin/gstack-state-root.sh'), path.join(path.dirname(helper), 'gstack-state-root.sh'));
|
||||
fs.copyFileSync(path.join(ROOT, 'bin/gstack-remote-identity.sh'), path.join(path.dirname(helper), 'gstack-remote-identity.sh'));
|
||||
for (const bin of ['gstack-paths', 'gstack-design-doc-find']) {
|
||||
fs.copyFileSync(path.join(ROOT, 'bin', bin), path.join(path.dirname(helper), bin));
|
||||
fs.chmodSync(path.join(path.dirname(helper), bin), 0o755);
|
||||
}
|
||||
const expected = path.join(home, '.gstack/projects/canonical-override/session-unknown-design-current.md');
|
||||
const wrong = path.join(home, '.gstack/projects/project/session-unknown-design-wrong.md');
|
||||
for (const file of [expected, wrong]) {
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const BIN = path.join(ROOT, 'bin/gstack-design-doc-find');
|
||||
const temps: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of temps.splice(0)) fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function fixture() {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'design-doc-find-'));
|
||||
temps.push(dir);
|
||||
const home = path.join(dir, 'home');
|
||||
const repo = path.join(dir, 'repo');
|
||||
fs.mkdirSync(home, { recursive: true });
|
||||
fs.mkdirSync(repo, { recursive: true });
|
||||
execFileSync('git', ['init', '-q', '-b', 'main'], { cwd: repo, timeout: 10_000 });
|
||||
const baseEnv = { PATH: process.env.PATH!, HOME: home, GIT_CEILING_DIRECTORIES: dir };
|
||||
const write = (file: string, ageSeconds: number) => {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, '# Design\n');
|
||||
const t = Date.now() / 1000 - ageSeconds;
|
||||
fs.utimesSync(file, t, t);
|
||||
return file;
|
||||
};
|
||||
const find = (env: Record<string, string> = {}, args = ['proj', 'feat-x']) =>
|
||||
execFileSync(BIN, args, { cwd: repo, env: { ...baseEnv, ...env }, encoding: 'utf8', timeout: 10_000 }).trim();
|
||||
return { dir, home, repo, write, find };
|
||||
}
|
||||
|
||||
describe('gstack-design-doc-find', () => {
|
||||
test('default root is ~/.gstack and prints nothing when no doc exists', () => {
|
||||
const f = fixture();
|
||||
expect(f.find()).toBe('');
|
||||
const doc = f.write(path.join(f.home, '.gstack/projects/proj/user-main-design-20260101.md'), 100);
|
||||
expect(f.find()).toBe(doc);
|
||||
});
|
||||
|
||||
test('GSTACK_HOME moves the state root', () => {
|
||||
const f = fixture();
|
||||
f.write(path.join(f.home, '.gstack/projects/proj/user-main-design-a.md'), 10);
|
||||
const custom = path.join(f.dir, 'custom-state');
|
||||
const doc = f.write(path.join(custom, 'projects/proj/user-main-design-b.md'), 100);
|
||||
expect(f.find({ GSTACK_HOME: custom })).toBe(doc);
|
||||
});
|
||||
|
||||
test('a branch-specific doc wins over a newer generic doc', () => {
|
||||
const f = fixture();
|
||||
const branch = f.write(path.join(f.home, '.gstack/projects/proj/user-feat-x-design-1.md'), 500);
|
||||
f.write(path.join(f.home, '.gstack/projects/proj/user-main-design-2.md'), 10);
|
||||
expect(f.find()).toBe(branch);
|
||||
expect(f.find({}, ['proj', 'other'])).toBe(path.join(f.home, '.gstack/projects/proj/user-main-design-2.md'));
|
||||
});
|
||||
|
||||
test('repo DESIGN.md wins only when at least as fresh, and beats docs/designs', () => {
|
||||
const f = fixture();
|
||||
const local = f.write(path.join(f.home, '.gstack/projects/proj/user-feat-x-design-1.md'), 100);
|
||||
const designs = f.write(path.join(f.repo, 'docs/designs/plan.md'), 10);
|
||||
expect(f.find()).toBe(designs);
|
||||
const top = f.write(path.join(f.repo, 'DESIGN.md'), 1000);
|
||||
expect(f.find()).toBe(local);
|
||||
fs.utimesSync(top, Date.now() / 1000, Date.now() / 1000);
|
||||
expect(f.find()).toBe(top);
|
||||
});
|
||||
|
||||
test('a repo doc is used when the state root has none', () => {
|
||||
const f = fixture();
|
||||
const designs = f.write(path.join(f.repo, 'docs/designs/plan.md'), 1000);
|
||||
expect(f.find()).toBe(designs);
|
||||
});
|
||||
|
||||
test('usage errors exit 2', () => {
|
||||
const f = fixture();
|
||||
expect(() => f.find({}, [])).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('rendered Design Doc Check block', () => {
|
||||
const rendered = fs.readFileSync(path.join(ROOT, 'plan-ceo-review/SKILL.md'), 'utf8');
|
||||
const block = rendered.slice(rendered.indexOf('DESIGN=$(~/.claude/skills/gstack/bin/gstack-design-doc-find'))
|
||||
.split('\n').slice(0, 2).join('\n');
|
||||
|
||||
test('the generated skill renders one helper invocation', () => {
|
||||
expect(block).toContain('gstack-design-doc-find "$SLUG" "$BRANCH"');
|
||||
expect(block).toContain('echo "Design doc found: $DESIGN"');
|
||||
expect(block).toContain('echo "No design doc found"');
|
||||
});
|
||||
|
||||
test('finds an office-hours doc under a custom GSTACK_HOME and keeps the default-root result', () => {
|
||||
const f = fixture();
|
||||
const installed = path.join(f.home, '.claude/skills/gstack/bin');
|
||||
fs.mkdirSync(installed, { recursive: true });
|
||||
for (const bin of ['gstack-design-doc-find', 'gstack-paths', 'gstack-state-root.sh']) {
|
||||
fs.copyFileSync(path.join(ROOT, 'bin', bin), path.join(installed, bin));
|
||||
fs.chmodSync(path.join(installed, bin), 0o755);
|
||||
}
|
||||
const run = (env: Record<string, string>) => execFileSync('bash', ['-c', `SLUG=proj; BRANCH=main; ${block}`],
|
||||
{ cwd: f.repo, env: { PATH: process.env.PATH!, HOME: f.home, GIT_CEILING_DIRECTORIES: f.dir, ...env }, encoding: 'utf8', timeout: 10_000 }).trim();
|
||||
const custom = path.join(f.dir, 'custom-state');
|
||||
const doc = f.write(path.join(custom, 'projects/proj/garry-main-design-20261003-120000.md'), 60);
|
||||
expect(run({ GSTACK_HOME: custom })).toBe(`Design doc found: ${doc}`);
|
||||
expect(run({})).toBe('No design doc found');
|
||||
const legacy = f.write(path.join(f.home, '.gstack/projects/proj/garry-main-design-20261001-120000.md'), 60);
|
||||
expect(run({})).toBe(`Design doc found: ${legacy}`);
|
||||
});
|
||||
});
|
||||
@@ -410,8 +410,9 @@ do not launch the downstream skill or open a browser.`,
|
||||
// + W1 guarded state-root resolution in the Context Recovery preamble, the
|
||||
// eureka log, the office-hours lookup and the taste-profile read; measured
|
||||
// 1.0834 (2026-09-30). + named design-direction defaults to avoid in the native
|
||||
// subagent prompt (s02 H4a); measured 1.0911 (2026-10-02).
|
||||
maxSizeRatio: 1.092,
|
||||
// subagent prompt (s02 H4a); measured 1.0911 (2026-10-02). Re-measured
|
||||
// 1.0758 (2026-10-03); cap = measured + 0.005 headroom.
|
||||
maxSizeRatio: 1.081,
|
||||
},
|
||||
cso: {
|
||||
skill: 'cso',
|
||||
@@ -537,7 +538,7 @@ do not launch the downstream skill or open a browser.`,
|
||||
},
|
||||
behavioral: 'prompt',
|
||||
maxSkeletonBytes: 74_500, // + Aside browser contract for Step 7 canary ({{ASIDE_SETUP}}); measured 73_523
|
||||
maxSizeRatio: 1.108, // + v1.81 Aside contract + gstack-browser fallback block; measured 1.077. + both untrusted-content marker formats in the browser fallback (W7f) + the readiness gate's project test command and live eval-store paths (s06 H6/H7); measured 1.1076 (2026-10-02).
|
||||
maxSizeRatio: 1.100, // + v1.81 Aside contract + gstack-browser fallback block; measured 1.077. + both untrusted-content marker formats in the browser fallback (W7f) + the readiness gate's project test command and live eval-store paths (s06 H6/H7); measured 1.1076 (2026-10-02). Re-measured 1.0943 (2026-10-03); cap = measured + 0.005 headroom.
|
||||
minUnionBytes: 91_000, // Phase 4 wave 1; estimated union ~94.9KB
|
||||
mustContain: ['readiness', 'merge', 'canary', 'revert', 'staging'],
|
||||
},
|
||||
|
||||
@@ -109,7 +109,7 @@ export const FILE_RETRY_BUDGETS = [
|
||||
{ file: 'test/skill-e2e-plan-eng-finding-floor.test.ts', attemptMs: PTY_MS, caseMs: PTY_MS },
|
||||
{ file: 'test/skill-e2e-plan-design-finding-floor.test.ts', attemptMs: PTY_MS, caseMs: PTY_MS },
|
||||
{ file: 'test/skill-e2e-plan-devex-finding-floor.test.ts', attemptMs: PTY_MS, caseMs: PTY_MS },
|
||||
{ file: 'test/skill-e2e-plan-mode-no-op.test.ts', attemptMs: 5 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS },
|
||||
{ file: 'test/skill-e2e-plan-mode-no-op.test.ts', attemptMs: 6 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS },
|
||||
{ file: 'test/skill-e2e-plan-ceo-mode-routing.test.ts', attemptMs: 2 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS },
|
||||
{ file: 'test/skill-e2e-plan-eng-plan-mode.test.ts', attemptMs: 2 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS },
|
||||
{ file: 'test/skill-e2e-plan-prosons.test.ts', attemptMs: 4 * (CAPTURE_MS + 10_000), caseMs: CAPTURE_MS + 10_000 },
|
||||
|
||||
@@ -23,4 +23,5 @@ export const OVERLAY_CASE_FILES: Record<string, string> = {
|
||||
'test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts': 'opus-4-7-effort-match-trivial',
|
||||
'test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation.test.ts': 'opus-4-7-literal-interpretation',
|
||||
'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts': 'claude-dedicated-tools-vs-bash-sonnet',
|
||||
'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts': 'claude-dedicated-tools-vs-bash-opus-5-5',
|
||||
};
|
||||
@@ -206,7 +206,13 @@ function deterministicPlanFloorFinding(input: PlanFloorReview): PlanFloorAssessm
|
||||
});
|
||||
}
|
||||
|
||||
/** Same warmup CLI, one turn and 30s cap as the replaced waiting-state judge.
|
||||
/** Per-call cap on the warmup-model assessment. Measured 2026-10-03 with
|
||||
* Claude Code 2.1.284 and claude-haiku-4-5 on the committed captured inputs:
|
||||
* 25 calls (10 sequential, 15 at 3-way concurrency), p50 14.7s, p95 27.4s,
|
||||
* max 31.3s, so the old 30s cap cut the tail. 90s is about 3x p95. */
|
||||
export const PLAN_FLOOR_ASSESSMENT_CAP_MS = 90_000;
|
||||
|
||||
/** Same warmup CLI and one turn as the replaced waiting-state judge.
|
||||
* The original case deadline bounds each call; complete input is never truncated. */
|
||||
export function judgePlanFloorReview(input: PlanFloorReview, opts: {
|
||||
binary: string; model: string; deadlineAt: number;
|
||||
@@ -225,7 +231,7 @@ export function judgePlanFloorReview(input: PlanFloorReview, opts: {
|
||||
try {
|
||||
const result = (opts.invoke ?? spawnSync)(opts.binary,
|
||||
['-p', '--model', opts.model, '--max-turns', '1'],
|
||||
{ input: prompt, stdio: ['pipe', 'pipe', 'pipe'], timeout: Math.min(30_000, remaining), encoding: 'utf8' });
|
||||
{ input: prompt, stdio: ['pipe', 'pipe', 'pipe'], timeout: Math.min(PLAN_FLOOR_ASSESSMENT_CAP_MS, remaining), encoding: 'utf8' });
|
||||
Object.assign(diagnostic, { rawOutput: String(result.stdout ?? ''), stderr: String(result.stderr ?? ''), status: result.status });
|
||||
if (result.error || result.status !== 0 || Date.now() >= opts.deadlineAt)
|
||||
throw Error(`Floor assessment did not complete: ${result.error?.message ?? `exit ${result.status}`} ${String(result.stderr ?? '').slice(-3000)}`.trim());
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* Structural reader for the shared QA exploratory "Names used below" list.
|
||||
* Tests check the commands built from these names (argument order, the CLI each
|
||||
* name is bound to) instead of pinning the names' spelling.
|
||||
*/
|
||||
export interface QaProbeNames {
|
||||
dir: string;
|
||||
deadline: string;
|
||||
guard: string;
|
||||
recorder: string;
|
||||
}
|
||||
|
||||
export function qaProbeNames(text: string): QaProbeNames {
|
||||
const start = text.indexOf('Names used below');
|
||||
if (start < 0) throw new Error('QA exploratory text has no "Names used below" list');
|
||||
const block = text.slice(start, text.indexOf('\n\n', start));
|
||||
const bound = (pattern: RegExp, what: string) => {
|
||||
const match = block.match(pattern);
|
||||
if (!match) throw new Error(`QA probe names: no ${what} binding in:\n${block}`);
|
||||
return match[1]!;
|
||||
};
|
||||
const dir = bound(/^- ([A-Z][A-Z_]+): [^\n]*probe directory/m, 'probe directory');
|
||||
const names = {
|
||||
dir,
|
||||
deadline: bound(new RegExp(`([A-Z][A-Z_]+): \`${dir}/deadline\\.json\``), 'deadline file'),
|
||||
guard: bound(/^- ([A-Z][A-Z_]+) = `[^`\n]*\/gstack-qa-deadline`/m, 'gstack-qa-deadline'),
|
||||
recorder: bound(/^- ([A-Z][A-Z_]+) = `[^`\n]*\/gstack-qa-evidence`/m, 'gstack-qa-evidence'),
|
||||
};
|
||||
if (new Set(Object.values(names)).size !== 4) throw new Error(`QA probe names must be distinct: ${JSON.stringify(names)}`);
|
||||
return names;
|
||||
}
|
||||
@@ -269,7 +269,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// (--max-concurrency, no retries), so worst-case cost is one pass of
|
||||
// each, sharing the API budget with sibling tests — not the
|
||||
// sequential ~+10min a local read suggests.
|
||||
'plan-mode-no-op': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'test/fixtures/auto-decide-recommendation-361c.json',
|
||||
'plan-mode-no-op': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'bin/gstack-design-doc-find', 'scripts/resolvers/design-doc-discovery.ts', 'scripts/resolvers/spec-review.ts', 'plan-devex-review/**', 'test/fixtures/auto-decide-recommendation-361c.json',
|
||||
|
||||
'test/fixtures/auto-decide-target-361c.json',
|
||||
|
||||
@@ -838,7 +838,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'test/fixtures/outside-async-task-m-events.json',
|
||||
'test/fixtures/autoplan-amend-input-77.json',
|
||||
'test/fixtures/autoplan-phase-handoff-6714.json','scripts/resolvers/learnings.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'test/fixtures/plan-scope-recovery-av.json', 'test/fixtures/design-scope-checkpoint-at.json', 'scripts/resolvers/composition.ts', 'autoplan/**', 'codex/**', 'bin/gstack-codex-probe', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'scripts/resolvers/design.ts', 'test/skill-e2e-autoplan-dual-voice.test.ts', 'bin/gstack-autoplan-snapshot.ts', 'test/fixtures/autoplan/t-ceo-omitted-obligations.json', 'test/fixtures/autoplan/u-ceo-original-loss.json', 'test/fixtures/autoplan/v-ceo-dangling-references.json',
|
||||
'scripts/resolvers/design-doc-discovery.ts', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/autoplan-phase-observer.ts', 'scripts/resolve-codex-generation-model.ts', 'scripts/resolvers/constants.ts'],
|
||||
'scripts/resolvers/design-doc-discovery.ts', 'bin/gstack-design-doc-find', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/autoplan-phase-observer.ts', 'scripts/resolve-codex-generation-model.ts', 'scripts/resolvers/constants.ts'],
|
||||
|
||||
// Multi-provider benchmark adapters — live API smoke against real claude/codex/gemini CLIs
|
||||
'benchmark-providers-live': ['bin/gstack-model-benchmark', 'test/helpers/providers/**', 'test/helpers/benchmark-runner.ts', 'test/helpers/pricing.ts', 'test/skill-e2e-benchmark-providers.test.ts'],
|
||||
@@ -1063,6 +1063,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'overlay-harness-opus-4-7-effort-match-trivial': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/fixtures/overlay-admission-child.ts', 'test/helpers/e2e-gate.ts'],
|
||||
'overlay-harness-opus-4-7-literal-interpretation': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/fixtures/overlay-admission-child.ts', 'test/helpers/e2e-gate.ts'],
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/fixtures/overlay-admission-child.ts', 'test/helpers/e2e-gate.ts'],
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/fixtures/overlay-admission-child.ts', 'test/helpers/e2e-gate.ts'],
|
||||
'journey-negatives': [
|
||||
|
||||
|
||||
@@ -1434,6 +1435,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic' | 'marathon'> = {
|
||||
'overlay-harness-opus-4-7-effort-match-trivial': 'periodic',
|
||||
'overlay-harness-opus-4-7-literal-interpretation': 'periodic',
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': 'periodic',
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5': 'periodic',
|
||||
'journey-negatives': 'periodic',
|
||||
};
|
||||
|
||||
@@ -1752,6 +1754,7 @@ export const E2E_KINDS: Record<string, 'rule' | 'behavior' | 'judge'> = {
|
||||
'overlay-harness-opus-4-7-effort-match-trivial': 'rule',
|
||||
'overlay-harness-opus-4-7-literal-interpretation': 'rule',
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': 'rule',
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash-opus-5-5': 'rule',
|
||||
'journey-negatives': 'rule',
|
||||
'review/SKILL.md workflow': 'judge',
|
||||
'setup-browser-cookies/SKILL.md workflow': 'judge',
|
||||
|
||||
@@ -169,10 +169,8 @@ const RATCHET_ALLOWLIST = JSON.parse(read('test/state-root-ratchet.allowlist.jso
|
||||
const readinessEvalReason = RATCHET_ALLOWLIST.find(entry => entry.path === 'land-and-deploy/sections/readiness-gate.md.tmpl')!.reason;
|
||||
const DESCRIPTION = 'user-facing description of where the data lives; no command reads or writes this literal';
|
||||
const SOURCE_COMMENT = 'source comment, not rendered into any skill';
|
||||
const DESIGN_DOC_HELD = 'held at the literal path: converting design-doc discovery to $GSTACK_STATE_ROOT made plan-eng-review ask its scope gate in prose instead of AskUserQuestion (plan-mode no-op eval: 5 of 5 runs failed with it, 6 of 6 passed without); see TODOS.md';
|
||||
|
||||
const STATE_ROOT_ALLOWLIST: Array<{ file: string; match: string; reason: string }> = [
|
||||
{ file: 'autoplan/SKILL.md.tmpl', match: 'ls -t ~/.gstack/projects/$SLUG/*-design-*.md', reason: DESIGN_DOC_HELD },
|
||||
{ file: 'autoplan/SKILL.md.tmpl', match: 'test plan on disk at ~/.gstack/projects/$SLUG/', reason: DESCRIPTION },
|
||||
{ file: 'careful/SKILL.md.tmpl', match: '`~/.gstack/projects/<slug>/careful-patterns.txt` (per-project)', reason: DESCRIPTION },
|
||||
{ file: 'design-shotgun/SKILL.md.tmpl', match: 'v1 schema at `~/.gstack/projects/$SLUG/taste-profile.json`', reason: DESCRIPTION },
|
||||
@@ -183,9 +181,6 @@ const STATE_ROOT_ALLOWLIST: Array<{ file: string; match: string; reason: string
|
||||
{ file: 'plan-ceo-review/sections/review-sections.md.tmpl', match: 'Keep in `~/.gstack/projects/` only (local, personal reference)', reason: DESCRIPTION },
|
||||
{ file: 'plan-tune/SKILL.md.tmpl', match: 'Logs stay local (`~/.gstack/projects/<slug>/question-log.jsonl`)', reason: DESCRIPTION },
|
||||
{ file: 'plan-tune/SKILL.md.tmpl', match: '`~/.gstack/projects/<slug>/question-log.jsonl` — nothing leaves your', reason: DESCRIPTION },
|
||||
{ file: 'scripts/resolvers/design-doc-discovery.ts', match: '* under ~/.gstack/projects/<slug>/, falling back', reason: SOURCE_COMMENT },
|
||||
{ file: 'scripts/resolvers/design-doc-discovery.ts', match: '_LOCALDOC=$(ls -t ~/.gstack/projects/$SLUG/*-$BRANCH-design-*.md', reason: DESIGN_DOC_HELD },
|
||||
{ file: 'scripts/resolvers/design-doc-discovery.ts', match: '_LOCALDOC=$(ls -t ~/.gstack/projects/$SLUG/*-design-*.md', reason: DESIGN_DOC_HELD },
|
||||
{ file: 'scripts/resolvers/learnings.ts', match: '* Learnings are stored per-project at ~/.gstack/projects/{slug}/learnings.jsonl', reason: SOURCE_COMMENT },
|
||||
{ file: 'scripts/resolvers/preamble.ts', match: 'local JSONL append to ~/.gstack/analytics/ (inline, inspectable)', reason: SOURCE_COMMENT },
|
||||
];
|
||||
|
||||
@@ -27,8 +27,8 @@ test('every fixture owns one paid wrapper; public models/trials/concurrency/turn
|
||||
}
|
||||
expect(isPaidTestFile('test/overlay-lifecycle.test.ts')).toBe(false);
|
||||
expect(OVERLAY_FIXTURES.every(f => f.trials === 10 && f.concurrency === 3)).toBe(true);
|
||||
expect(OVERLAY_FIXTURES.map(f => f.maxTurns ?? 5)).toEqual([15,8,15,15]);
|
||||
expect(OVERLAY_FIXTURES.map(f => f.model)).toEqual([...Array(3).fill('claude-opus-4-7'), 'claude-sonnet-4-6']);
|
||||
expect(OVERLAY_FIXTURES.map(f => f.maxTurns ?? 5)).toEqual([15,8,15,15,15]);
|
||||
expect(OVERLAY_FIXTURES.map(f => f.model)).toEqual([...Array(3).fill('claude-opus-4-7'), 'claude-sonnet-4-6', 'claude-opus-5-5']);
|
||||
expect([OVERLAY_CASE_WORK_MS, OVERLAY_RECORD_GRACE_MS, OVERLAY_CASE_OUTER_MS, OVERLAY_MIN_FILE_WALL_MS]).toEqual([1_800_000, 5_000, 1_810_000, 1_830_000]);
|
||||
});
|
||||
|
||||
|
||||
@@ -255,11 +255,12 @@ describe('literal fixture task correctness', () => {
|
||||
expect(() => assertReadOnlyWorkspace(before, snapshotWorkspace(dir))).toThrow('b');
|
||||
}));
|
||||
test('registry retires absent-nudge fanout cases and preserves remaining model/trial budgets', () => {
|
||||
expect(OVERLAY_FIXTURES).toHaveLength(4);
|
||||
expect(OVERLAY_FIXTURES).toHaveLength(5);
|
||||
expect(OVERLAY_FIXTURES.some((f) => f.id.includes('fanout') || f.comparison?.unsupportedHypothesis)).toBe(false);
|
||||
expect(OVERLAY_FIXTURES.every((f) => f.trials === 10 && f.comparison)).toBe(true);
|
||||
expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-4-7')).toHaveLength(3);
|
||||
expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-sonnet-4-6')).toHaveLength(1);
|
||||
expect(OVERLAY_FIXTURES.filter((f) => f.model === 'claude-opus-5-5')).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ describe('overlay file policy', () => {
|
||||
});
|
||||
|
||||
test('only the exact wrapper family gets one attempt and the extra process grace', () => {
|
||||
expect(overlayFiles).toHaveLength(4);
|
||||
expect(overlayFiles).toHaveLength(5);
|
||||
expect(OVERLAY_MAX_ACTIVE_SHARDS).toBe(1);
|
||||
expect(OVERLAY_MIN_FILE_WALL_MS).toBe(1_830_000);
|
||||
for (const file of overlayFiles) {
|
||||
@@ -125,16 +125,16 @@ describe('overlay file policy', () => {
|
||||
});
|
||||
|
||||
describe('overlay manifest affinity and CI capacity', () => {
|
||||
test('actual lifecycle wrapper guards include all four in periodic and exclude all four from gate', () => {
|
||||
test('actual lifecycle wrapper guards include all five in periodic and exclude all five from gate', () => {
|
||||
for (const tier of ['periodic', 'gate'] as const) {
|
||||
const manifest = buildRunManifest({ tier, sliceCount: 6, evalsAll: true, env: { EVALS_ALL: '1' } });
|
||||
const entries = manifest.entries.filter(entry => isOverlayTestFile(entry.file));
|
||||
expect(entries).toHaveLength(4);
|
||||
expect(entries).toHaveLength(5);
|
||||
expect(entries.every(entry => entry.status === (tier === 'periodic' ? 'planned' : 'excluded'))).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('93 files retain every case, reserve slice six, and fit 330 minutes with actual family walls', () => {
|
||||
test('94 files retain every case, reserve slice six, and fit 330 minutes with actual family walls', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'overlay-affinity-'));
|
||||
const normalFiles = Array.from({ length: 89 }, (_, i) => `test/skill-e2e-normal-${i.toString().padStart(2, '0')}.test.ts`);
|
||||
const discovered = [...normalFiles, ...overlayFiles];
|
||||
@@ -146,11 +146,11 @@ describe('overlay manifest affinity and CI capacity', () => {
|
||||
}
|
||||
const opts = { tier: 'periodic' as const, sliceCount: 6, evalsAll: true, discovered, rootDir: dir, env: { EVALS_ALL: '1' } };
|
||||
const manifest = buildRunManifest(opts);
|
||||
expect(manifest.entries).toHaveLength(93);
|
||||
expect(new Set(manifest.entries.map(e => e.file)).size).toBe(93);
|
||||
expect(manifest.entries).toHaveLength(94);
|
||||
expect(new Set(manifest.entries.map(e => e.file)).size).toBe(94);
|
||||
expect(manifest.entries.every(e => e.status === 'planned')).toBe(true);
|
||||
const counts = [1, 2, 3, 4, 5, 6].map(slice => manifest.entries.filter(e => e.slice === slice).length);
|
||||
expect(counts).toEqual([18, 18, 18, 18, 17, 4]);
|
||||
expect(counts).toEqual([18, 18, 18, 18, 17, 5]);
|
||||
expect(manifest.entries.filter(e => e.slice === 6).map(e => e.file).sort()).toEqual([...overlayFiles].sort());
|
||||
expect(buildRunManifest({ ...opts, discovered: [...discovered].reverse() })).toEqual(manifest);
|
||||
expect(parseRunManifest(JSON.stringify(manifest))).toEqual(manifest);
|
||||
@@ -175,8 +175,8 @@ describe('overlay manifest affinity and CI capacity', () => {
|
||||
const overlayMinutes = Math.ceil(overlayFiles.length / OVERLAY_MAX_ACTIVE_SHARDS)
|
||||
* Math.max(...overlayFiles.map(file => resolvePaidShardTimeoutMs([file]))) / 60_000;
|
||||
expect(normalMinutes).toBe(270);
|
||||
expect(overlayMinutes).toBe(122);
|
||||
// The CI budget plan (what the workflow runs) keeps the four overlays
|
||||
expect(overlayMinutes).toBe(152.5);
|
||||
// The CI budget plan (what the workflow runs) keeps the five overlays
|
||||
// in one final one-at-a-time slice and its job cap covers them.
|
||||
const budget = buildRunManifest({ ...opts, sliceCount: undefined, sliceBudgetMs: 540_000, jobs });
|
||||
const lastSlice = budget.entries.filter(e => e.slice === budget.sliceCount).map(e => e.file).sort();
|
||||
|
||||
@@ -18,7 +18,7 @@ import { resolveModuleSelection } from './helpers/e2e-helpers';
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const CEO_FILES = [
|
||||
'test/skill-e2e-plan.test.ts', 'test/skill-e2e-ask-user-question-format-compliance.test.ts',
|
||||
'test/skill-e2e-retro.test.ts', 'test/skill-llm-eval.test.ts',
|
||||
'test/skill-e2e-retro.test.ts', 'test/skill-llm-eval.test.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts',
|
||||
];
|
||||
const ceoManifest = () => buildRunManifest({ tier: 'gate', profile: 'pr', sliceCount: 1,
|
||||
evalsAll: false, env: {}, changedFiles: ['plan-ceo-review/SKILL.md.tmpl'], discovered: CEO_FILES });
|
||||
@@ -64,15 +64,16 @@ describe('PR profile paid-runner integration', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('planner binds E2E and judge IDs and explicitly defers direct-describe broad probes', () => {
|
||||
test('planner binds E2E and judge IDs and plans the CEO plan-mode probe in the PR lane', () => {
|
||||
const manifest = ceoManifest();
|
||||
expect(manifest.profile).toBe('pr');
|
||||
expect(manifest.selection?.e2e).toContain('auq-format-gate');
|
||||
expect(manifest.selection?.e2e).not.toContain('plan-ceo-review-plan-mode');
|
||||
expect(manifest.selection?.e2e).toContain('plan-ceo-review-plan-mode');
|
||||
expect(manifest.selection?.judges).toContain('plan-ceo-review/SKILL.md modes');
|
||||
expect(manifest.entries.find(entry => entry.file.includes('skill-e2e-retro'))?.status).toBe('skipped-by-diff');
|
||||
expect(manifest.entries.find(entry => entry.file.includes('ask-user-question'))?.status).toBe('planned');
|
||||
expect(manifest.prCoverage?.deferred.some(item => item.id === 'plan-ceo-review-plan-mode')).toBe(true);
|
||||
expect(manifest.prCoverage?.deferred.some(item => item.id === 'plan-ceo-review-plan-mode')).toBe(false);
|
||||
expect(manifest.entries.find(entry => entry.file === 'test/skill-e2e-plan-ceo-plan-mode.test.ts')?.status).toBe('planned');
|
||||
expect(parseRunManifest(JSON.stringify(manifest))).toEqual(manifest);
|
||||
const env = paidSelectionEnv('pr', manifest.selection!, manifest.selectionReason);
|
||||
expect(JSON.parse(env.EVALS_SELECTION_JSON!).selected).toEqual(manifest.selection!.e2e);
|
||||
|
||||
@@ -30,7 +30,7 @@ const expectedWalls = {
|
||||
'test/skill-e2e-plan-eng-finding-floor.test.ts': 1_020_000,
|
||||
'test/skill-e2e-plan-design-finding-floor.test.ts': 1_020_000,
|
||||
'test/skill-e2e-plan-devex-finding-floor.test.ts': 1_020_000,
|
||||
'test/skill-e2e-plan-mode-no-op.test.ts': 3_120_000,
|
||||
'test/skill-e2e-plan-mode-no-op.test.ts': 3_720_000,
|
||||
'test/skill-e2e-plan-ceo-mode-routing.test.ts': 1_320_000,
|
||||
'test/skill-e2e-plan-eng-plan-mode.test.ts': 1_320_000,
|
||||
'test/skill-e2e-plan-prosons.test.ts': 1_360_000,
|
||||
@@ -88,7 +88,7 @@ test('source allowances retain all captures, cases, and finalization grace', ()
|
||||
if (file.endsWith('-finding-floor')) expect(read(`test/skill-e2e-${file}.test.ts`)).toContain('timeoutMs: CAPTURE_LONG_MS');
|
||||
}
|
||||
const noop = read('test/skill-e2e-plan-mode-no-op.test.ts');
|
||||
expect(noop).toContain("['plan-ceo-review', 'plan-eng-review', 'plan-design-review'] as const");
|
||||
expect(noop).toContain("['plan-ceo-review', 'plan-eng-review', 'plan-design-review', 'plan-devex-review'] as const");
|
||||
expect(noop).toContain("['plan-eng-review', 'plan-design-review'] as const");
|
||||
expect(timeoutExpressions('test/skill-e2e-plan-mode-no-op.test.ts')).toEqual(Array(2).fill('CAPTURE_LONG_MS'));
|
||||
expect(read('test/skill-e2e-plan-ceo-mode-routing.test.ts').match(/^\s*{ mode: '/gm)).toHaveLength(2);
|
||||
@@ -210,7 +210,7 @@ test('detached PR fallback and release commands cover their actual default worke
|
||||
const prFloor = Math.ceil((Math.ceil(fullGateFiles.length / prWorkers) * 1_800_000 + fullGateFiles.reduce(
|
||||
(total, file) => total + Math.max(0, resolvePaidShardBudget([file]).timeoutMs - 1_800_000), 0,
|
||||
)) / 1000 * 1.05);
|
||||
expect(prFloor).toBe(72_755);
|
||||
expect(prFloor).toBe(73_385);
|
||||
expect(prWall).toBe(92_820_000);
|
||||
expect(prWall).toBeGreaterThanOrEqual(paidShardWallUpperBoundMs(files, prWorkers) + 120_000);
|
||||
|
||||
@@ -229,8 +229,8 @@ test('detached PR fallback and release commands cover their actual default worke
|
||||
)) / 1000 * 1.05));
|
||||
}
|
||||
const detachedReleaseWall = Number(scripts['eval:bg:release'].match(/--timeout (\d+)/)?.[1]) * 1000;
|
||||
expect(releaseFloors).toEqual([21_725, 35_711]);
|
||||
expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(57_436);
|
||||
expect(releaseFloors).toEqual([22_355, 35_742]);
|
||||
expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(58_097);
|
||||
expect(detachedReleaseWall).toBe(116_700_000);
|
||||
expect(detachedReleaseWall).toBeGreaterThanOrEqual(releaseWall + 120_000);
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {expect,test,spyOn} from 'bun:test';
|
||||
import {buildPlanFloorReviewPrompt,validatePlanFloorAssessment,resolvePlanFloorCitations,judgePlanFloorReview,pickPlanFloorMode,pickPlanFloorProductType,type PlanFloorReview} from './helpers/plan-floor-review';
|
||||
import {buildPlanFloorReviewPrompt,validatePlanFloorAssessment,resolvePlanFloorCitations,judgePlanFloorReview,pickPlanFloorMode,pickPlanFloorProductType,PLAN_FLOOR_ASSESSMENT_CAP_MS,type PlanFloorReview} from './helpers/plan-floor-review';
|
||||
import {FORCING_FLOOR_CEO, FORCING_FLOOR_DEVEX} from './fixtures/forcing-finding-seeds';
|
||||
import capturedQuotes from './fixtures/plan-floor-quote-70b.json';
|
||||
import productTypes from './fixtures/plan-floor-product-type-70b.json';
|
||||
@@ -57,14 +57,15 @@ test('large complete input is preserved; over-limit input is rejected without in
|
||||
expect(()=>judgePlanFloorReview(input,{binary:'fake',model:'warmup',deadlineAt:Date.now()+30_000,invoke:(()=>{calls++;throw Error('must not execute');}) as any})).toThrow();
|
||||
expect(calls).toBe(0);
|
||||
});
|
||||
test('replacement judge retains the original CLI model, one turn, 30s cap and absolute case deadline',()=>{
|
||||
for(const remaining of [5000,60_000]){
|
||||
test('replacement judge retains the original CLI model, one turn, measured cap and absolute case deadline',()=>{
|
||||
expect(PLAN_FLOOR_ASSESSMENT_CAP_MS).toBeGreaterThan(30_000);
|
||||
for(const remaining of [5000,60_000,PLAN_FLOOR_ASSESSMENT_CAP_MS+60_000]){
|
||||
const deadlineAt=Date.now()+remaining;let calls=0;
|
||||
const actual=judgePlanFloorReview(review(),{binary:'/fake/claude',model:'unchanged-warmup',deadlineAt,invoke:((file,args,opts)=>{
|
||||
calls++;expect(file).toBe('/fake/claude');expect(args).toEqual(['-p','--model','unchanged-warmup','--max-turns','1']);
|
||||
expect(opts.stdio).toEqual(['pipe','pipe','pipe']);expect(opts.encoding).toBe('utf8');
|
||||
expect(opts.input).toBe(buildPlanFloorReviewPrompt(review()));
|
||||
expect(opts.timeout).toBeGreaterThan(0);expect(opts.timeout).toBeLessThanOrEqual(Math.min(30_000,remaining));
|
||||
expect(opts.timeout).toBeGreaterThan(0);expect(opts.timeout).toBeLessThanOrEqual(Math.min(PLAN_FLOOR_ASSESSMENT_CAP_MS,remaining));
|
||||
return {status:0,stdout:JSON.stringify(citationFinding()),stderr:''};
|
||||
}) as any});expect(calls).toBe(1);expect(actual.kind).toBe('finding');
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import { generateQAExploratory } from '../scripts/resolvers/qa';
|
||||
import { generateTestBootstrap } from '../scripts/resolvers/testing';
|
||||
import { HOST_PATHS } from '../scripts/resolvers/types';
|
||||
import { runBashScript } from './helpers/bash-script';
|
||||
import { qaProbeNames } from './helpers/qa-probe-names';
|
||||
|
||||
const ctx = { host: 'claude', skillName: 'qa', tmplPath: '', paths: HOST_PATHS.claude };
|
||||
const method = generateQAMethodology(ctx);
|
||||
@@ -149,16 +150,17 @@ describe('compact QA browser recipes retain native operations', () => {
|
||||
test('bounded exploration rechecks the clock around checkpoints without replacing probe evidence', () => {
|
||||
for (const skillName of ['qa', 'qa-only', 'review', 'ship']) {
|
||||
const loop = generateQAExploratory({ ...ctx, skillName });
|
||||
const n = qaProbeNames(loop);
|
||||
for (const contract of [
|
||||
'bun G start D SECONDS [EARLIER_UTC]',
|
||||
`bun ${n.guard} start ${n.deadline} SECONDS [EARLIER_UTC]`,
|
||||
'Set SECONDS to the shorter mode/caller limit',
|
||||
'G enforces the deadline',
|
||||
`${n.guard} enforces the deadline`,
|
||||
'QA_DEADLINE receipts are not observations',
|
||||
'Never reset D/bypass G',
|
||||
'Report refusals as not-run',
|
||||
'Bounded browsers: `bun G run D -- COMMAND ARGS`',
|
||||
`\`bun ${n.guard} run ${n.deadline} -- COMMAND ARGS\``,
|
||||
'announce finite command timeouts',
|
||||
]) expect(loop).toContain(contract);
|
||||
expect(loop).toMatch(new RegExp(`never reset ${n.deadline}\\W+bypass ${n.guard}`, 'i'));
|
||||
for (const field of ['observationCommand', 'observed', 'hypothesis', 'nextCommand']) expect(loop).toContain(`${field}:`);
|
||||
expect(loop).toContain('Functional Full, Quick and Regression have no default total timer');
|
||||
if (skillName !== 'qa-only') expect(loop).toContain('Explicit plan checks and revalidation remain required beyond this smoke budget');
|
||||
|
||||
@@ -4,6 +4,7 @@ import { join } from 'node:path';
|
||||
import { ALL_HOST_CONFIGS } from '../hosts';
|
||||
import { RESOLVERS } from '../scripts/resolvers';
|
||||
import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types';
|
||||
import { qaProbeNames } from './helpers/qa-probe-names';
|
||||
|
||||
const root = join(import.meta.dir, '..');
|
||||
const callers = ['qa', 'qa-only', 'review', 'ship'];
|
||||
@@ -76,9 +77,11 @@ describe('QA caller authority in pure host renders', () => {
|
||||
for (const caller of callers) {
|
||||
const body = RESOLVERS.QA_EXPLORATORY(context(host.name, caller));
|
||||
expect(body).toContain('The **caller** (/qa, /qa-only, /review or /ship)');
|
||||
expect(body).toContain('bun G start D SECONDS [EARLIER_UTC]');
|
||||
expect(body).toContain('G enforces the deadline');
|
||||
expect(body.indexOf('bun G start D')).toBeLessThan(body.indexOf('1. First demonstrate success'));
|
||||
const n = qaProbeNames(body);
|
||||
const start = `bun ${n.guard} start ${n.deadline} SECONDS [EARLIER_UTC]`;
|
||||
expect(body).toContain(start);
|
||||
expect(body).toContain(`${n.guard} enforces the deadline`);
|
||||
expect(body.indexOf(start)).toBeLessThan(body.indexOf('1. First demonstrate success'));
|
||||
expect(body).toMatch(/stop when scoped contracts are tested or blocked/i);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -15,6 +15,7 @@ import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { readQACheckpointFiles } from './helpers/qa-checkpoint-evidence';
|
||||
import { generateQAExploratory, generateQAResource, generateQAReview, generateQAReviewPreflight } from '../scripts/resolvers/qa';
|
||||
import { HOST_PATHS } from '../scripts/resolvers/types';
|
||||
import { qaProbeNames } from './helpers/qa-probe-names';
|
||||
|
||||
function nativeCall(id: string, name: string, input: object, output: string, parent: string | null = null, failed = false) {
|
||||
return [
|
||||
@@ -673,10 +674,12 @@ describe('generated actual parent paths', () => {
|
||||
expect(text).not.toContain('nest unchanged child JSON');
|
||||
});
|
||||
test('the shared smoke has explicit limits without waiving required plan checks', () => {
|
||||
const body = fs.readFileSync(path.join(import.meta.dir, '../qa/sections/exploratory.md'), 'utf8').replace(/\s+/g, ' ');
|
||||
const raw = fs.readFileSync(path.join(import.meta.dir, '../qa/sections/exploratory.md'), 'utf8');
|
||||
const body = raw.replace(/\s+/g, ' ');
|
||||
expect(body).toMatch(/stop after 5 minutes or 12 probes/i);
|
||||
expect(body).toContain('G enforces the deadline');
|
||||
expect(body).toMatch(/never reset D\/bypass G/i);
|
||||
const n = qaProbeNames(raw);
|
||||
expect(body).toContain(`${n.guard} enforces the deadline`);
|
||||
expect(body).toMatch(new RegExp(`never reset ${n.deadline}\\W+bypass ${n.guard}`, 'i'));
|
||||
expect(body).toMatch(/plan checks and revalidation remain required beyond this smoke budget/i);
|
||||
expect(body).toContain('leaves /review incomplete');
|
||||
expect(body).toMatch(/\/ship blocked unless the user explicitly accepts that named risk/i);
|
||||
|
||||
+18
-11
@@ -5,6 +5,9 @@ import { ALL_HOST_CONFIGS } from '../hosts';
|
||||
import { generateQAExploratory, generateQAMethodReads, generateQAReview, generateQAReviewPreflight } from '../scripts/resolvers/qa';
|
||||
import { generatePlanVerificationExec } from '../scripts/resolvers/plan-gates';
|
||||
import { HOST_PATHS } from '../scripts/resolvers/types';
|
||||
import { qaProbeNames } from './helpers/qa-probe-names';
|
||||
|
||||
const startCommand = (text: string) => { const n = qaProbeNames(text); return `bun ${n.guard} start ${n.deadline} SECONDS`; };
|
||||
|
||||
function assertPreparation(text: string) {
|
||||
expect(text).toMatch(/reads in order before writing charters or probing/i);
|
||||
@@ -12,7 +15,7 @@ function assertPreparation(text: string) {
|
||||
expect(text).toMatch(/do not repeat a Read already completed/i);
|
||||
const stages = ['Read `sections/scope.md`', '**Functional surfaces:**', 'Read `sections/system-functional.md`',
|
||||
'**Browser surfaces only:**', 'Read `sections/qa-patterns.md`', '## 1. Charter and preflight',
|
||||
'bun G start D SECONDS', '1. First demonstrate success'];
|
||||
startCommand(text), '1. First demonstrate success'];
|
||||
const positions = stages.map(stage => text.indexOf(stage));
|
||||
expect(positions.every(position => position >= 0)).toBe(true);
|
||||
expect(positions).toEqual([...positions].sort((a, b) => a - b));
|
||||
@@ -26,13 +29,13 @@ function assertBoundsAndLayout(text: string) {
|
||||
for (const rule of [
|
||||
/functional full, quick and regression have no default total timer/i,
|
||||
/shorter mode\/caller limit/i,
|
||||
/without a total time limit, do not start D/i,
|
||||
new RegExp(`without a total time limit, do not (?:start|create) ${qaProbeNames(text).deadline}`, 'i'),
|
||||
/finite command timeouts/i,
|
||||
/mixed standalone runs use REPORT_DIR\/browser and REPORT_DIR\/functional/i,
|
||||
/one final report at REPORT_DIR/i,
|
||||
/caller paths win/i,
|
||||
]) expect(text).toMatch(rule);
|
||||
expect(text.search(/caller paths win/i)).toBeLessThan(text.indexOf('bun G start D SECONDS'));
|
||||
expect(text.search(/caller paths win/i)).toBeLessThan(text.indexOf(startCommand(text)));
|
||||
}
|
||||
|
||||
function assertPlanExecution(text: string, shared = generateQAExploratory({ host: 'claude', skillName: 'qa', tmplPath: '', paths: HOST_PATHS.claude })) {
|
||||
@@ -127,7 +130,7 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
'QA_DEADLINE receipts are not observations',
|
||||
]) expect(text).toContain(contract);
|
||||
assertBoundsAndLayout(text);
|
||||
expect(text.indexOf('Browser Quick: SECONDS=30')).toBeLessThan(text.indexOf('bun G start D'));
|
||||
expect(text.indexOf('Browser Quick: SECONDS=30')).toBeLessThan(text.indexOf(startCommand(text)));
|
||||
}
|
||||
});
|
||||
|
||||
@@ -152,7 +155,7 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
} else {
|
||||
expect(text).toMatch(/preserve every safe program-JSON key\/value and identity hash unchanged/i);
|
||||
}
|
||||
expect(text).toMatch(/Q supplies observed; never transcribe it/i);
|
||||
expect(text).toMatch(new RegExp(`${qaProbeNames(text).recorder} supplies observed; never transcribe it`, 'i'));
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -163,15 +166,19 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
const steps = text.slice(text.indexOf('1. First demonstrate success')).split(/\n(?=[2-5]\. )/);
|
||||
expect(steps).toHaveLength(5);
|
||||
expect(text).toContain('/gstack-qa-deadline');
|
||||
expect(text).toContain('bun G start D SECONDS [EARLIER_UTC]');
|
||||
expect(text).toContain('Bounded browsers: `bun G run D -- COMMAND ARGS`');
|
||||
expect(text).toMatch(/never reset D\/bypass G/i);
|
||||
expect(text).toMatch(/invalid\/missing D stops probes/i);
|
||||
const n = qaProbeNames(text);
|
||||
for (const name of Object.values(n)) expect(name.length).toBeGreaterThan(3);
|
||||
expect(text).toContain(`bun ${n.guard} start ${n.deadline} SECONDS [EARLIER_UTC]`);
|
||||
expect(text).toContain(`\`bun ${n.guard} run ${n.deadline} -- COMMAND ARGS\``);
|
||||
expect(text).toContain(`\`bun ${n.recorder} capture ${n.dir} NNN [--public] --deadline ${n.deadline} -- COMMAND ARGS\``);
|
||||
expect(text).toContain(`\`bun ${n.recorder} materialize ${n.dir} annotations.json\``);
|
||||
expect(text).toMatch(new RegExp(`never reset ${n.deadline}\\W+bypass ${n.guard}`, 'i'));
|
||||
expect(text).toMatch(new RegExp(`invalid/missing ${n.deadline} stops probes`, 'i'));
|
||||
expect(steps[0]).toMatch(/demonstrate success: output and durable effects/i);
|
||||
expect(steps[1]).toContain('`bun G status D`');
|
||||
expect(steps[1]).toContain(`\`bun ${n.guard} status ${n.deadline}\``);
|
||||
expect(steps[1].indexOf('If expired')).toBeLessThan(steps[1].indexOf('**Publish before probing.**'));
|
||||
expect(steps[1]).toMatch(/\bstop exploration\b[^.]*write the report/i);
|
||||
expect(steps[2]).toContain('G enforces the deadline');
|
||||
expect(steps[2]).toContain(`${n.guard} enforces the deadline`);
|
||||
expect(steps[2]).toMatch(/refusals as not-run/i);
|
||||
expect(steps[3]).toContain('via steps 2–3');
|
||||
expect(steps[4]).toContain('return to step 2');
|
||||
|
||||
@@ -310,7 +310,6 @@ const GENERATED_WITH_GUIDANCE = [
|
||||
// guidance and its bounded worker policy is specified in its own skeleton.
|
||||
'design-consultation/sections/proposal-and-preview.md',
|
||||
'design-review/SKILL.md',
|
||||
'design-shotgun/SKILL.md',
|
||||
'document-release/sections/release-body.md',
|
||||
'office-hours/SKILL.md',
|
||||
'office-hours/sections/design-and-handoff.md',
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { registerOverlayCase } from './helpers/overlay-case';
|
||||
|
||||
describeE2ETier('periodic')('overlay behavior contract v2 (SDK)', () => {
|
||||
registerOverlayCase('claude-dedicated-tools-vs-bash-opus-5-5');
|
||||
});
|
||||
@@ -21,11 +21,16 @@
|
||||
* ends in 'asked', the question that fired must be the scope gate itself
|
||||
* (outside plan mode with no named target, the gate is the FIRST
|
||||
* question by contract).
|
||||
* - plan-devex-review: no no-target scope gate, so only the terminal
|
||||
* outcome and the absent plan-mode reminder are asserted. It reviews a
|
||||
* pasted developer-facing plan: without one it reviews the checkout's
|
||||
* branch diff, and a tests-only diff legitimately ends at its
|
||||
* applicability gate, which is neither 'asked' nor 'plan_ready'.
|
||||
* - named-target case: a pasted draft (initialPlanContent) IS an
|
||||
* explicitly-named target, so the gate must NOT ask — and the review
|
||||
* must actually consume the pasted content.
|
||||
*
|
||||
* Cost note: 5 sequential PTY runs (~3-5 min each) in the gate lane, up
|
||||
* Cost note: 6 sequential PTY runs (~3-5 min each) in the gate lane, up
|
||||
* from 1 pre-bypass. Selected only when plan-ceo/eng/design or the runner
|
||||
* change (see 'plan-mode-no-op' in touchfiles.ts).
|
||||
*/
|
||||
@@ -57,8 +62,21 @@ weekly export emails. One new component, one route, one test file.
|
||||
- test/${SEED_TOKEN}.test.tsx (new)
|
||||
`;
|
||||
|
||||
const DEVEX_PLAN = `
|
||||
# Plan: tasks export command
|
||||
|
||||
## Scope
|
||||
Add a \`tasks export --format csv|json\` CLI subcommand for developers who
|
||||
script against the task tracker. Prints to stdout; --out writes a file.
|
||||
|
||||
## Developer experience
|
||||
- New flag help text and a README quickstart example.
|
||||
- Exit code 2 with a one-line error for an unknown --format value.
|
||||
`;
|
||||
|
||||
describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
|
||||
for (const skillName of ['plan-ceo-review', 'plan-eng-review', 'plan-design-review'] as const) {
|
||||
for (const skillName of ['plan-ceo-review', 'plan-eng-review', 'plan-design-review', 'plan-devex-review'] as const) {
|
||||
const hasScopeGate = skillName === 'plan-eng-review' || skillName === 'plan-design-review';
|
||||
test(`${skillName} reaches a terminal outcome outside plan mode`, async () => {
|
||||
const obs = await runPlanSkillObservation({
|
||||
skillName,
|
||||
@@ -74,9 +92,10 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
|
||||
// renders as a lettered menu, so a judge 'waiting' verdict on a
|
||||
// spinner-only frame must not end the run as 'asked' before that
|
||||
// menu has rendered.
|
||||
...(skillName === 'plan-ceo-review'
|
||||
? {}
|
||||
: { extraArgs: ['--disallowedTools', 'AskUserQuestion'], requireProseEvidence: true }),
|
||||
...(hasScopeGate
|
||||
? { extraArgs: ['--disallowedTools', 'AskUserQuestion'], requireProseEvidence: true }
|
||||
: {}),
|
||||
...(skillName === 'plan-devex-review' ? { initialPlanContent: DEVEX_PLAN } : {}),
|
||||
});
|
||||
|
||||
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
|
||||
@@ -94,7 +113,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
|
||||
// section is leaking outside plan mode.
|
||||
expect(obs.evidence).not.toContain(PLAN_MODE_REMINDER);
|
||||
|
||||
if (skillName !== 'plan-ceo-review') {
|
||||
if (hasScopeGate) {
|
||||
// Scope-gate bypass must not misfire: no auto-select announcement
|
||||
// outside plan mode.
|
||||
expect(obs.scopeGateAutoSelectObserved ?? false).toBe(false);
|
||||
|
||||
Reference in new issue
Block a user