Merge remote-tracking branch 'origin/main' into ponytail-inspiration-review

# Conflicts:
#	CHANGELOG.md
#	TODOS.md
#	VERSION
#	docs/TESTING_INTERNALS.md
#	package.json
#	scripts/test-free-shards.ts
#	test/helpers/llm-judge.ts
#	test/helpers/touchfiles-data.ts
This commit is contained in:
Garry Tan
2026-08-29 16:11:58 +00:00
173 changed files with 6796 additions and 1346 deletions
+7 -6
View File
@@ -13,6 +13,8 @@
import { describe, test, expect } from 'bun:test';
import { spawnSync } from 'child_process';
import { runBin } from './helpers/run-bin';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
@@ -21,16 +23,15 @@ const ROOT = path.resolve(import.meta.dir, '..');
const BIN = path.join(ROOT, 'bin', 'gstack-model-benchmark');
function run(args: string[], opts: { env?: Record<string, string> } = {}): { status: number | null; stdout: string; stderr: string } {
const result = spawnSync('bun', ['run', BIN, ...args], {
const result = runBin('bun', ['run', BIN, ...args], {
cwd: ROOT,
env: { ...process.env, ...opts.env },
encoding: 'utf-8',
timeout: 15000,
env: opts.env,
timeoutMs: 15000,
});
return {
status: result.status,
stdout: result.stdout?.toString() ?? '',
stderr: result.stderr?.toString() ?? '',
stdout: result.stdout,
stderr: result.stderr,
};
}
+76
View File
@@ -0,0 +1,76 @@
/**
* One Bun version across every CI surface.
*
* The drift class this pins: Dockerfile.ci's comment records that the old
* `| BUN_VERSION=x.y.z bash` form silently installed latest on every image
* rebuild (observed 1.3.13/1.3.14 drift vs the 1.3.10 devs ran locally),
* and before 2026-08-29 the lanes disagreed four ways (1.3.13 / latest /
* unpinned / 1.3.10). Different Bun versions change test-runner OUTPUT
* SHAPES the strict classifiers regex-match, spawn semantics, and shell
* parsing — a lane on a different Bun is testing a different product.
*
* Bumping Bun: change every surface in one commit; this test names each one.
*/
import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
const ROOT = path.resolve(__dirname, '..');
const WORKFLOWS_DIR = path.join(ROOT, '.github', 'workflows');
interface Pin {
surface: string;
version: string;
}
function collectPins(): Pin[] {
const pins: Pin[] = [];
for (const name of fs.readdirSync(WORKFLOWS_DIR).sort()) {
if (!/\.ya?ml$/.test(name)) continue;
const source = fs.readFileSync(path.join(WORKFLOWS_DIR, name), 'utf-8');
const lines = source.split('\n');
for (let i = 0; i < lines.length; i++) {
if (!/uses:\s*oven-sh\/setup-bun@/.test(lines[i])) continue;
// A pinned stanza is `with:` + `bun-version: <v>` within the next few
// lines; an unpinned setup-bun is itself drift (installs latest).
const window = lines.slice(i + 1, i + 4).join('\n');
const m = window.match(/bun-version:\s*["']?([\w.]+)["']?/);
pins.push({
surface: `${name}:${i + 1}`,
version: m ? m[1] : '<unpinned setup-bun — installs latest>',
});
}
}
const dockerfile = fs.readFileSync(
path.join(ROOT, '.github', 'docker', 'Dockerfile.ci'), 'utf-8');
const dockerPin = dockerfile.match(/bash -s ["']?bun-v([\w.]+)["']?/);
pins.push({
surface: 'Dockerfile.ci',
version: dockerPin ? dockerPin[1] : '<no bun-vX.Y.Z positional arg>',
});
const gitlab = fs.readFileSync(path.join(ROOT, '.gitlab-ci.yml'), 'utf-8');
const gitlabPin = gitlab.match(/BUN_VERSION:\s*["']?([\w.]+)["']?/);
pins.push({
surface: '.gitlab-ci.yml',
version: gitlabPin ? gitlabPin[1] : '<no BUN_VERSION>',
});
return pins;
}
describe('bun version pins', () => {
test('every CI surface pins the same bun version', () => {
const pins = collectPins();
// Sanity: the scan found the known surfaces (a regex rot that finds
// nothing must fail loudly, not vacuously pass).
expect(pins.length).toBeGreaterThanOrEqual(6);
const versions = [...new Set(pins.map((p) => p.version))];
const detail = pins.map((p) => `${p.surface}${p.version}`).join('\n');
expect(versions, `bun version drift across CI surfaces:\n${detail}`).toHaveLength(1);
expect(versions[0]).toMatch(/^\d+\.\d+\.\d+$/);
});
});
+2 -1
View File
@@ -20,6 +20,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { setupSkillDir, skillFromWorktree, captureSectionReads } from './helpers/auq-sdk-capture';
import { CARVE_GUARDS } from './helpers/carve-guards';
@@ -97,7 +98,7 @@ describeE2E('carve behavioral section-loading (periodic, SDK capture)', () => {
});
expect(output.trim().length).toBeGreaterThan(200);
},
540_000,
CAPTURE_LONG_MS,
);
}
});
+17 -25
View File
@@ -15,14 +15,15 @@
* `description: |` block (multi-line) instead of the trim'd one-line
* `description: ...(gstack)` form.
*
* The smoke test mutates the working tree mid-run. It restores the default
* trim'd state in a finally block so a crash mid-test still leaves a clean
* working tree.
* The smoke test renders the full-catalog variant into an isolated
* --out-dir — the working tree is never written, so there is no restore
* pass (and no half-restored tree if the test crashes mid-run).
*/
import { describe, test, expect } from 'bun:test';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
const REPO_ROOT = path.resolve(import.meta.dir, '..');
@@ -58,24 +59,26 @@ describe('--catalog-mode=full opt-out wiring (static)', () => {
describe('--catalog-mode=full opt-out behavior (smoke)', () => {
test('--catalog-mode=full produces multi-line description in frontmatter', () => {
// Save the trim'd state so we can restore it.
const trimmedShip = fs.readFileSync(SHIP_SKILL, 'utf-8');
// The TRACKED ship/SKILL.md carries the default trim'd form (read-only check).
// #1778: the trimmed ship description has an interior colon ("Ship workflow:")
// and is now YAML-quoted — tolerate the optional surrounding quotes.
const trimmedShip = fs.readFileSync(SHIP_SKILL, 'utf-8');
expect(trimmedShip).toMatch(/^description: "?Ship workflow:[^\n]*\(gstack\)"?\n/m);
const outDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-catalog-full-'));
try {
// Run with --catalog-mode=full. Mutates working tree.
const result = spawnSync('bun', ['run', 'gen:skill-docs', '--catalog-mode=full'], {
// Render --catalog-mode=full into an isolated out-dir. The working
// tree is never written, so no restore pass is needed.
const result = spawnSync('bun', ['run', 'gen:skill-docs', '--catalog-mode=full', '--out-dir', outDir], {
cwd: REPO_ROOT,
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 60_000,
});
expect(result.status).toBe(0);
// After --catalog-mode=full, frontmatter description is the legacy
// In the full-mode render, frontmatter description is the legacy
// multi-line block, not the trim'd one-line form.
const fullShip = fs.readFileSync(SHIP_SKILL, 'utf-8');
const fullShip = fs.readFileSync(path.join(outDir, 'ship', 'SKILL.md'), 'utf-8');
expect(fullShip).toMatch(/^description: \|\s*$/m); // YAML block scalar
// Legacy multi-line content includes "Use when asked to..." in the
// frontmatter (in trim mode this lives in the body section).
@@ -87,23 +90,12 @@ describe('--catalog-mode=full opt-out behavior (smoke)', () => {
// (because the routing prose stayed in frontmatter).
const body = fullShip.slice(fmEnd);
expect(body).not.toContain('## When to invoke this skill');
// Non-mutation proof: the tracked ship/SKILL.md is byte-unchanged —
// a catalog-mode render must never rewrite the committed trim'd state.
expect(fs.readFileSync(SHIP_SKILL, 'utf-8')).toBe(trimmedShip);
} finally {
// Restore default trim state regardless of test outcome.
const restore = spawnSync('bun', ['run', 'gen:skill-docs'], {
cwd: REPO_ROOT,
stdio: ['ignore', 'pipe', 'pipe'],
timeout: 60_000,
});
if (restore.status !== 0) {
// eslint-disable-next-line no-console
console.error(
'CRITICAL: failed to restore default trim state. Run `bun run gen:skill-docs` to clean up.',
);
}
// Sanity-check the restored state matches what we saw at the start.
const restoredShip = fs.readFileSync(SHIP_SKILL, 'utf-8');
// #1778: restored trim state has the YAML-quoted (interior-colon) description.
expect(restoredShip).toMatch(/^description: "?Ship workflow:[^\n]*\(gstack\)"?\n/m);
fs.rmSync(outDir, { recursive: true, force: true });
}
}, 180_000);
+36
View File
@@ -0,0 +1,36 @@
/**
* The CI image tag is a content hash computed independently in three
* workflows — evals.yml, evals-periodic.yml, ci-image.yml — and they were
* synced by comment only (filed in TODOS.md as the "three-way image-tag
* drift" gap). If one file's hashFiles() input list drifts, that workflow
* computes a DIFFERENT tag for the same content: the eval lanes stop finding
* the prebuilt image and silently rebuild it on every run (minutes per run,
* no red check), or ci-image prebuilds a tag nobody looks up.
*/
import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
const ROOT = path.resolve(__dirname, '..');
const FILES = ['evals.yml', 'evals-periodic.yml', 'ci-image.yml'];
function hashFilesCalls(name: string): string[] {
const source = fs.readFileSync(
path.join(ROOT, '.github', 'workflows', name), 'utf-8');
// Only tag-computation sites: hashFiles() inside a `tag=` output line.
return [...source.matchAll(/tag=[^\n]*?(hashFiles\([^)]*\))/g)].map((m) => m[1]);
}
describe('ci image tag binding', () => {
test('all three workflows compute the tag from the identical hashFiles() input list', () => {
const perFile = FILES.map((f) => ({ file: f, calls: hashFilesCalls(f) }));
for (const { file, calls } of perFile) {
// Each workflow computes the tag exactly once; zero means the scan
// regex rotted (must fail loudly, not vacuously pass).
expect(calls, `${file}: expected exactly one tag hashFiles() site`).toHaveLength(1);
}
const expressions = [...new Set(perFile.map((p) => p.calls[0]))];
const detail = perFile.map((p) => `${p.file}${p.calls[0]}`).join('\n');
expect(expressions, `image-tag hashFiles() drift:\n${detail}`).toHaveLength(1);
});
});
+205
View File
@@ -0,0 +1,205 @@
/**
* bin/gstack-code-intelligence — CLI surface smoke tests.
*
* lib/code-intelligence/* is covered by test/code-intelligence.test.ts, which
* also drives the CLI's `index` and `search` consent/policy refusal paths.
* This file covers the argument-handling surface those tests skip: usage on
* bad/missing subcommands, `select` and `consent` validation + state writes,
* and the `suggest` offer gate — all hermetic under a mkdtemp GSTACK_HOME
* (the selection store lives at $GSTACK_HOME/code-intelligence.json), and all
* on paths that never call detectAvailable(), so nothing probes providers or
* the network.
*
* Note: the CLI has no `--help` flag — every unrecognized action (including
* `--help`) routes to the usage message on stderr with exit 1. Pinned below.
*/
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { runBin } from './helpers/run-bin';
const ROOT = path.resolve(import.meta.dir, '..');
const CLI = path.join(ROOT, 'bin', 'gstack-code-intelligence');
let home: string;
let workDir: string;
beforeEach(() => {
home = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-cli-home-'));
workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-cli-work-'));
});
afterEach(() => {
fs.rmSync(home, { recursive: true, force: true });
fs.rmSync(workDir, { recursive: true, force: true });
});
function runCli(...args: string[]) {
return runBin('bun', [CLI, ...args], { cwd: workDir, gstackHome: home, home });
}
function readStore(): { provider: string | null; consents: Record<string, boolean>; declined: boolean } {
return JSON.parse(fs.readFileSync(path.join(home, 'code-intelligence.json'), 'utf-8'));
}
describe('gstack-code-intelligence: usage surface', () => {
test('no arguments: usage on stderr, exit 1', () => {
const result = runCli();
expect(result.status).toBe(1);
expect(result.stderr).toContain('gstack-code-intelligence:');
expect(result.stderr).toContain('Usage:');
expect(result.stderr).toContain('select <provider>');
expect(result.stdout).toBe('');
});
test('unknown subcommand: usage on stderr, exit 1', () => {
const result = runCli('frobnicate');
expect(result.status).toBe(1);
expect(result.stderr).toContain('Usage:');
});
test('--help has no exit-0 handler — it routes to the usage failure (current behavior)', () => {
const result = runCli('--help');
expect(result.status).toBe(1);
expect(result.stderr).toContain('Usage:');
});
});
describe('gstack-code-intelligence: select', () => {
test('invalid provider is rejected with the select usage line', () => {
const result = runCli('select', 'bogus-provider');
expect(result.status).toBe(1);
expect(result.stderr).toContain('Usage: select <gbrain|sourcebot|graphify|none>');
expect(fs.existsSync(path.join(home, 'code-intelligence.json'))).toBe(false);
});
test('select with no argument is rejected the same way', () => {
const result = runCli('select');
expect(result.status).toBe(1);
expect(result.stderr).toContain('Usage: select <gbrain|sourcebot|graphify|none>');
});
test('select none records the decline so the offer is never repeated', () => {
const result = runCli('select', 'none');
expect(result.status).toBe(0);
expect(result.stdout).toContain('declined');
expect(result.stdout).toContain('will not ask again');
const store = readStore();
expect(store.provider).toBeNull();
expect(store.declined).toBe(true);
});
test('selecting the local provider persists it without an off-machine warning', () => {
const result = runCli('select', 'graphify');
expect(result.status).toBe(0);
expect(result.stdout).toContain('selected Graphify.');
expect(result.stdout).not.toContain('off this machine');
const store = readStore();
expect(store.provider).toBe('graphify');
expect(store.declined).toBe(false);
});
test('selecting a non-local provider warns that content leaves the machine', () => {
const result = runCli('select', 'gbrain');
expect(result.status).toBe(0);
expect(result.stdout).toContain('selected GBrain.');
expect(result.stdout).toContain('off this machine');
expect(readStore().provider).toBe('gbrain');
});
});
describe('gstack-code-intelligence: consent', () => {
test('the yes/no value is required — a bare path records NOTHING', () => {
const result = runCli('consent', workDir);
expect(result.status).toBe(1);
expect(result.stderr).toContain('never assumed');
expect(fs.existsSync(path.join(home, 'code-intelligence.json'))).toBe(false);
});
test('an unknown value records NOTHING', () => {
const result = runCli('consent', workDir, 'maybe');
expect(result.status).toBe(1);
expect(result.stderr).toContain('never assumed');
expect(fs.existsSync(path.join(home, 'code-intelligence.json'))).toBe(false);
});
test('consent yes persists true for the resolved repo path', () => {
const result = runCli('consent', workDir, 'yes');
expect(result.status).toBe(0);
expect(result.stdout).toContain('indexing consent recorded');
expect(readStore().consents[fs.realpathSync(workDir)] ?? readStore().consents[workDir]).toBe(true);
});
test('consent no persists an explicit DENIED — a "no" is a durable answer too', () => {
const result = runCli('consent', workDir, 'no');
expect(result.status).toBe(0);
expect(result.stdout).toContain('DENIED');
expect(readStore().consents[fs.realpathSync(workDir)] ?? readStore().consents[workDir]).toBe(false);
});
test('consent with no path defaults to the cwd', () => {
const result = runCli('consent', 'yes');
expect(result.status).toBe(0);
const consents = readStore().consents;
const keys = Object.keys(consents);
expect(keys.length).toBe(1);
// resolve(cwd) — the child's cwd is workDir (possibly via a symlinked tmp).
expect([workDir, fs.realpathSync(workDir)]).toContain(keys[0]);
expect(consents[keys[0]]).toBe(true);
});
});
describe('gstack-code-intelligence: suggest (offer gate)', () => {
test('a non-repo directory never triggers the offer (--json)', () => {
const result = runCli('suggest', workDir, '--json');
expect(result.status).toBe(0);
const parsed = JSON.parse(result.stdout);
expect(parsed.offer).toBe(false);
expect(parsed.reason).toBe('not-a-repo');
expect(parsed.fileCount).toBeNull();
expect([workDir, fs.realpathSync(workDir)]).toContain(parsed.repoPath);
});
test('a selected provider suppresses the offer before any repo probing', () => {
expect(runCli('select', 'graphify').status).toBe(0);
const result = runCli('suggest', workDir, '--json');
expect(result.status).toBe(0);
const parsed = JSON.parse(result.stdout);
expect(parsed.offer).toBe(false);
expect(parsed.reason).toBe('provider-selected');
});
test('an explicit decline suppresses the offer permanently', () => {
expect(runCli('select', 'none').status).toBe(0);
const result = runCli('suggest', workDir, '--json');
expect(result.status).toBe(0);
expect(JSON.parse(result.stdout).reason).toBe('declined');
});
test('human-readable no-offer output names the reason', () => {
const result = runCli('suggest', workDir);
expect(result.status).toBe(0);
expect(result.stdout).toContain('no offer (not-a-repo)');
});
});
describe('gstack-code-intelligence: provider-requiring commands without a selection', () => {
test('index refuses when no provider is selected', () => {
const result = runCli('index', workDir);
expect(result.status).toBe(1);
expect(result.stderr).toContain('no provider selected');
});
test('search refuses when no provider is selected', () => {
const result = runCli('search', 'anything');
expect(result.status).toBe(1);
expect(result.stderr).toContain('no provider selected');
});
test('search with no query prints the search usage', () => {
const result = runCli('search');
expect(result.status).toBe(1);
expect(result.stderr).toContain('Usage: search <query...>');
});
});
+15 -9
View File
@@ -26,6 +26,7 @@
* Periodic tier (Codex non-determinism). Cost: ~$2-3 per full run.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runCodexSkill, installSkillToTempHome } from './helpers/codex-session-runner';
import type { CodexResult } from './helpers/codex-session-runner';
import { EvalCollector } from './helpers/eval-store';
@@ -47,7 +48,12 @@ const CODEX_AVAILABLE = (() => {
} catch { return false; }
})();
const evalsEnabled = !!process.env.EVALS;
const SKIP = !CODEX_AVAILABLE || !evalsEnabled;
// External-service test — periodic tier only (CLAUDE.md tiering rule 3),
// matching codex-e2e.test.ts / codex-e2e-sol-scope.test.ts. Without this
// guard the sharded runner's "no whole-file tier guard" default would run
// Codex spawns in the GATE tier on every PR.
const tierOk = process.env.EVALS_TIER === 'periodic';
const SKIP = !CODEX_AVAILABLE || !evalsEnabled || !tierOk;
const describeCodex = SKIP ? describe.skip : describe;
// --- Touchfiles ---
@@ -181,7 +187,7 @@ describeCodex('Codex Plan Format — CEO Mode Selection', () => {
const result = await runCodexSkill({
skillDir,
prompt: `Read the plan-ceo-review skill. Read plan.md (the plan to review). Proceed to Step 0F (Mode Selection) where the skill presents 4 mode options (SCOPE EXPANSION, SELECTIVE EXPANSION, HOLD SCOPE, SCOPE REDUCTION) via AskUserQuestion. These options differ in kind (review posture), not coverage. ${captureInstruction(outFile)}`,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
cwd: planDir,
skillName: 'gstack-plan-ceo-review',
sandbox: 'workspace-write',
@@ -203,7 +209,7 @@ describeCodex('Codex Plan Format — CEO Mode Selection', () => {
// kind-differentiated: no fabricated score, must have note
expect(captured).not.toMatch(COMPLETENESS_RE);
expect(captured).toMatch(KIND_NOTE_RE);
}, 360_000);
}, CAPTURE_LONG_MS);
});
describeCodex('Codex Plan Format — CEO Approach Menu', () => {
@@ -221,7 +227,7 @@ describeCodex('Codex Plan Format — CEO Approach Menu', () => {
const result = await runCodexSkill({
skillDir,
prompt: `Read the plan-ceo-review skill. Read plan.md. Proceed to Step 0C-bis (Implementation Alternatives / Approach Menu) where the skill generates 2-3 approaches (minimal viable vs ideal architecture) and presents them via AskUserQuestion. These options differ in coverage so Completeness: N/10 applies. ${captureInstruction(outFile)}`,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
cwd: planDir,
skillName: 'gstack-plan-ceo-review',
sandbox: 'workspace-write',
@@ -240,7 +246,7 @@ describeCodex('Codex Plan Format — CEO Approach Menu', () => {
expect(captured.length).toBeGreaterThan(ELI10_LENGTH_FLOOR);
expect(captured).toMatch(RECOMMENDATION_RE);
expect(captured).toMatch(COMPLETENESS_RE);
}, 360_000);
}, CAPTURE_LONG_MS);
});
describeCodex('Codex Plan Format — Eng Coverage Issue', () => {
@@ -258,7 +264,7 @@ describeCodex('Codex Plan Format — Eng Coverage Issue', () => {
const result = await runCodexSkill({
skillDir,
prompt: `Read the plan-eng-review skill. Read plan.md. In your Section 3 Test Review, generate ONE AskUserQuestion about test coverage depth where options are clearly coverage-differentiated: A) full coverage incl. edge + error paths (Completeness 10/10), B) happy path only (7/10), C) smoke test (3/10). ${captureInstruction(outFile)}`,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
cwd: planDir,
skillName: 'gstack-plan-eng-review',
sandbox: 'workspace-write',
@@ -277,7 +283,7 @@ describeCodex('Codex Plan Format — Eng Coverage Issue', () => {
expect(captured.length).toBeGreaterThan(ELI10_LENGTH_FLOOR);
expect(captured).toMatch(RECOMMENDATION_RE);
expect(captured).toMatch(COMPLETENESS_RE);
}, 360_000);
}, CAPTURE_LONG_MS);
});
describeCodex('Codex Plan Format — Eng Kind Issue', () => {
@@ -295,7 +301,7 @@ describeCodex('Codex Plan Format — Eng Kind Issue', () => {
const result = await runCodexSkill({
skillDir,
prompt: `Read the plan-eng-review skill. Read plan.md. In your Section 1 Architecture review, generate ONE AskUserQuestion about an architectural choice where the options differ in kind (e.g. Redis vs Postgres materialized view vs in-process cache — different kinds of systems with different tradeoffs, NOT more-or-less-complete versions of the same thing). ${captureInstruction(outFile)}`,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
cwd: planDir,
skillName: 'gstack-plan-eng-review',
sandbox: 'workspace-write',
@@ -316,5 +322,5 @@ describeCodex('Codex Plan Format — Eng Kind Issue', () => {
// kind-differentiated: no fabricated score
expect(captured).not.toMatch(COMPLETENESS_RE);
expect(captured).toMatch(KIND_NOTE_RE);
}, 360_000);
}, CAPTURE_LONG_MS);
});
@@ -21,6 +21,7 @@
* Periodic tier (Codex non-determinism, ~$2-3/run).
*/
import { describe, test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import * as path from 'node:path';
import { e2eTierEnabled } from './helpers/e2e-gate';
import { runCodexSkill } from './helpers/codex-session-runner';
@@ -69,7 +70,7 @@ describeCodex('/codex recommendation substance (live, periodic)', () => {
skillDir: path.join(ROOT, 'codex'),
skillName: 'codex',
prompt: FIXTURE_DIFF,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (result.output.startsWith('SKIP:')) {
@@ -98,6 +99,6 @@ describeCodex('/codex recommendation substance (live, periodic)', () => {
);
}
},
360_000,
CAPTURE_LONG_MS,
);
});
+5 -1
View File
@@ -11,6 +11,7 @@
* golden), parallel shards (worktree copies), or live symlinked installs.
*/
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
@@ -125,6 +126,9 @@ describeSol('GPT-5.6 Sol full-artifact scope termination', () => {
const generated = spawnSync(
'bun',
['run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--model', 'gpt-5.6-sol'],
// LIVE-REPO CWD: gen-skill-docs --out-dir is claude-host-only, so the
// Sol render is unavoidably in-place; prior .agents tree is snapshotted
// above and restored below.
{ cwd: ROOT, encoding: 'utf8', timeout: 120_000 },
);
if (generated.status !== 0) {
@@ -259,5 +263,5 @@ You are authorized to implement the minimal fix. The task boundary is src/parse-
expect(readmeDecoyUntouched).toBe(true);
console.log(`codex-sol-scope: ${result.tokens} tokens, ${result.toolCalls.length} tool calls, ${Math.round(result.durationMs / 1000)}s`);
}, 300_000);
}, CAPTURE_MS);
});
+5 -4
View File
@@ -14,6 +14,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runCodexSkill, parseCodexJSONL, installSkillToTempHome } from './helpers/codex-session-runner';
import type { CodexResult } from './helpers/codex-session-runner';
import { CODEX_REVIEW_E2E_SECTIONS } from './helpers/skill-fixture';
@@ -150,7 +151,7 @@ describeCodex('Codex E2E', () => {
const result = await runCodexSkill({
skillDir,
prompt: 'List any skills or instructions you have available. Just list the names.',
timeoutMs: 60_000,
timeoutMs: JUDGE_MS,
cwd: testWorktree,
skillName: 'gstack-review',
});
@@ -171,7 +172,7 @@ describeCodex('Codex E2E', () => {
expect(
outputLower.includes('review') || outputLower.includes('gstack') || outputLower.includes('skill'),
).toBe(true);
}, 120_000);
}, JUDGE_MS);
// Validates that Codex can invoke the gstack-review skill, run a diff-based
// code review, and produce structured review output with findings/issues.
@@ -186,7 +187,7 @@ describeCodex('Codex E2E', () => {
const result = await runCodexSkill({
skillDir,
prompt: 'Run the gstack-review skill on this repository. Review the current branch diff and report your findings.',
timeoutMs: 540_000,
timeoutMs: CAPTURE_LONG_MS,
cwd: testWorktree,
skillName: 'gstack-review',
sections: CODEX_REVIEW_E2E_SECTIONS,
@@ -224,5 +225,5 @@ describeCodex('Codex E2E', () => {
outputLower.includes('p1') ||
outputLower.includes('p2');
expect(hasReviewContent).toBe(true);
}, 600_000);
}, CAPTURE_LONG_MS);
});
+26 -42
View File
@@ -15,47 +15,31 @@ import * as fs from 'fs';
import * as path from 'path';
const ROOT = path.resolve(import.meta.dir, '..');
const SKILL_GLOBS = [
'plan-ceo-review',
'plan-eng-review',
'plan-design-review',
'plan-devex-review',
'office-hours',
'codex',
'investigate',
'qa',
'retro',
'cso',
'review',
'ship',
'design-review',
'devex-review',
'qa-only',
'design-consultation',
'design-shotgun',
'autoplan',
'land-and-deploy',
'plan-tune',
'document-release',
'context-save',
'context-restore',
'health',
'setup-deploy',
'setup-browser-cookies',
'canary',
'learn',
'benchmark',
'benchmark-models',
'make-pdf',
'open-gstack-browser',
'gstack-upgrade',
'pair-agent',
'design-html',
'freeze',
'unfreeze',
'careful',
'guard',
];
/**
* Every top-level skill directory with a SKILL.md.tmpl, discovered from
* disk. This replaced a hand-maintained 39-name list that had drifted to
* 39-of-54 templates on disk — none of the unlisted 15 happened to be
* interactive, so there was no LIVE gap, but the next interactive skill
* would have landed unguarded with no signal.
*/
function skillTemplateDirs(): string[] {
return fs.readdirSync(ROOT, { withFileTypes: true })
.filter((entry) => {
// Directory symlinks (connect-chrome → open-gstack-browser) count too:
// existsSync below follows them, and duplicates only re-check the same
// template. isDirectory() is false for symlinked dirs, hence statSync.
if (entry.name.startsWith('.') || entry.name === 'node_modules') return false;
try {
return fs.statSync(path.join(ROOT, entry.name)).isDirectory()
&& fs.existsSync(path.join(ROOT, entry.name, 'SKILL.md.tmpl'));
} catch {
return false;
}
})
.map((entry) => entry.name)
.sort();
}
/**
* Load .tmpl files for each skill and return the names of those that have
@@ -63,7 +47,7 @@ const SKILL_GLOBS = [
*/
function findInteractiveSkills(): string[] {
const interactive: string[] = [];
for (const skill of SKILL_GLOBS) {
for (const skill of skillTemplateDirs()) {
const tmplPath = path.join(ROOT, skill, 'SKILL.md.tmpl');
if (!fs.existsSync(tmplPath)) continue;
const content = fs.readFileSync(tmplPath, 'utf-8');
+68 -9
View File
@@ -11,9 +11,15 @@
* Mapping rule (test filenames do NOT map mechanically to tier keys): for each
* `test/skill-e2e-*.test.ts` with an EVALS_TIER self-gate, search the
* E2E_TOUCHFILES / LLM_JUDGE_TOUCHFILES dep lists for the exact file path. If
* found under key K, the file's self-gate tier must equal E2E_TIERS[K]. Files
* not named in any dep list are REPORTED as unmapped (a nudge to add them to
* their eval's dep list), never silently skipped.
* found under key K, the file's self-gate tier must equal E2E_TIERS[K].
*
* Self-registration is a HARD invariant (the dep-list sweep): every
* skill-e2e file must be named in at least one touchfiles dep list, so that
* editing only the test's prompt/assertions diff-selects the test itself.
* Before the sweep, 129 of ~177 E2E keys did not list their own declaring
* file — a changed test never re-ran on its own change. Files that genuinely
* cannot be mapped (no E2E map key exists for them) sit in KNOWN_UNREGISTERED
* below; that set is a ratchet, it only shrinks.
*/
import { describe, test, expect } from 'bun:test';
@@ -35,6 +41,26 @@ const SELF_GATE_RE = /EVALS_TIER\s*===\s*['"](gate|periodic)['"]/g;
// the declared tier exactly like the raw predicate's tier literal did.
const HELPER_GATE_RE = /\b(?:describeE2ETier|e2eTierEnabled)\(\s*['"](gate|periodic)['"]/g;
/**
* Ratchet, not amnesty (same contract as KNOWN_MATRIX_GAPS in
* test/evals-workflow-matrix.test.ts): skill-e2e files that are named in NO
* touchfiles dep list because no E2E map key exists for them. Every entry
* carries a one-line reason. Do NOT add new files here — give the test an
* E2E map key (touchfiles + tier) and register the file in its dep list.
* A stale entry (file deleted, or file now registered) FAILS the suite —
* delete it. Target: empty set.
*/
const KNOWN_UNREGISTERED = new Set([
// Standalone periodic self-gated probe; template-literal testNames (auq-consistency-${i}), no E2E map key — fail-open-safe, runs on every periodic sweep.
'test/skill-e2e-auq-consistency.test.ts',
// Standalone periodic self-gated matrix; template-literal testNames (auq-matrix-${m.skill}), no E2E map key — fail-open-safe, runs on every periodic sweep.
'test/skill-e2e-auq-matrix.test.ts',
// Standalone periodic self-gated A/B probe; template-literal testNames (auq-ab-${label}), no E2E map key — fail-open-safe, runs on every periodic sweep.
'test/skill-e2e-auq-verbose-vs-carved-ab.test.ts',
// bin-script pipeline test (spawns bun scripts, no model spend) that lives under the skill-e2e-* glob; no E2E map key exists for it.
'test/skill-e2e-memory-pipeline.test.ts',
]);
describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () => {
const testFiles = readdirSync(TEST_DIR)
.filter((f) => f.startsWith('skill-e2e-') && f.endsWith('.test.ts'))
@@ -44,14 +70,28 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () =>
test('every self-gated test file named in a dep list matches its declared tier', () => {
const misaligned: string[] = [];
const unregistered: string[] = [];
const reported: string[] = [];
for (const file of testFiles) {
const content = readFileSync(path.join(TEST_DIR, file), 'utf-8');
const repoPath = `test/${file}`;
// HARD self-registration invariant, independent of self-gate shape:
// a skill-e2e file named in no dep list means editing the test itself
// selects nothing — the changed test never re-runs on its own change.
const owningKeys = Object.keys(allDeps).filter((k) => allDeps[k].includes(repoPath));
if (owningKeys.length === 0 && !KNOWN_UNREGISTERED.has(repoPath)) {
unregistered.push(
`${repoPath}: not named in any touchfiles dep list — editing this test file would `
+ 'never diff-select it. Add the file path to its E2E map key\'s dep list in '
+ 'test/helpers/touchfiles-data.ts (do NOT extend KNOWN_UNREGISTERED for new files).',
);
}
const tiers = new Set<string>();
for (const m of content.matchAll(SELF_GATE_RE)) tiers.add(m[1]);
for (const m of content.matchAll(HELPER_GATE_RE)) tiers.add(m[1]);
const repoPath = `test/${file}`;
if (tiers.size === 0) {
// Every skill-e2e file is expected to self-gate; zero matches means
// either a genuinely ungated file or a gate shape the regex can't
@@ -65,8 +105,9 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () =>
}
const selfTier = [...tiers][0];
const owningKeys = Object.keys(allDeps).filter((k) => allDeps[k].includes(repoPath));
if (owningKeys.length === 0) {
// Only KNOWN_UNREGISTERED files reach here (anything else already
// hard-failed above) — keep the visible nudge.
reported.push(`${repoPath} (self-gates '${selfTier}'): not named in any touchfiles dep list`);
continue;
}
@@ -86,18 +127,36 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () =>
}
}
// Reported, not asserted: coverage holes the invariant can see but not
// arbitrate. Add the test file to its eval's dep list (or a tier entry
// for the key) to bring it under the invariant.
// Reported, not asserted: tier-observability holes the invariant can see
// but not arbitrate (map-driven files legitimately have no whole-file
// self-gate; ratcheted files stay visible). Registration itself is
// asserted below.
if (reported.length > 0) {
console.warn(
`[tier-alignment] ${reported.length} file(s) outside the invariant:\n ` + reported.join('\n '),
`[tier-alignment] ${reported.length} file(s) outside the tier invariant:\n ` + reported.join('\n '),
);
}
expect(unregistered).toEqual([]);
expect(misaligned).toEqual([]);
});
// Ratchet cleanup enforcement (same contract as evals-workflow-matrix's
// burn-down test): a KNOWN_UNREGISTERED entry whose file was deleted, or
// whose file is now named in a dep list, is stale — delete the entry so
// the set can only shrink.
test('KNOWN_UNREGISTERED holds only live, still-unregistered files', () => {
const stale = [...KNOWN_UNREGISTERED].filter((repoPath) => {
const file = repoPath.replace(/^test\//, '');
if (!testFiles.includes(file)) return true; // file gone
return Object.keys(allDeps).some((k) => allDeps[k].includes(repoPath)); // now registered
});
expect(
stale,
'Entry registered in a dep list or file removed — delete it from KNOWN_UNREGISTERED.',
).toEqual([]);
});
// HARD invariant (C6): the paid sharded runner skips a skill-e2e shard when
// none of the file's MAPPED test names (E2E map keys quoted in its source,
// union E2E map keys whose dep list registers the file) are diff-selected.
+62
View File
@@ -0,0 +1,62 @@
/**
* Two invariants over paid-test timeout policy:
*
* 1. FIT: every tier in test/helpers/eval-budgets.ts executes inside the
* sharded runner's wall with real overhead (bun startup + module load +
* reporting). A budget the wall kills first is fiction — the failure
* surfaces as a shard 'timed-out' (no bun summary, no per-test message)
* instead of a clean test timeout. This is the structural fix for the
* seven 1,700s-inside-a-1,500s-job literals found in the 2026-08 audit.
*
* 2. RATCHET: raw numeric timeout literals in paid test files only shrink.
* New tests use the tiers; a literal is legal only with justification,
* and the count is pinned so sprawl can't regrow.
*/
import { describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { ALL_TIERS, PTY_LONG_MS } from './helpers/eval-budgets';
import { isPaidTestFile } from './helpers/paid-test-set';
import { DEFAULT_SHARD_TIMEOUT_MS } from '../scripts/test-paid-shards';
const ROOT = path.resolve(__dirname, '..');
/** Wall overhead reserve: bun startup, module load, retry bookkeeping. */
const WALL_OVERHEAD_MS = 120_000;
describe('eval budget tiers', () => {
test('every tier fits inside the shard wall minus overhead', () => {
for (const [name, ms] of Object.entries(ALL_TIERS)) {
expect(ms, `${name} exceeds the shard wall minus overhead`)
.toBeLessThanOrEqual(DEFAULT_SHARD_TIMEOUT_MS - WALL_OVERHEAD_MS);
}
});
test('tiers are ordered and the ceiling is PTY_LONG', () => {
const values = Object.values(ALL_TIERS);
expect([...values].sort((a, b) => a - b)).toEqual(values);
expect(Math.max(...values)).toBe(PTY_LONG_MS);
});
test('no paid-test timeout literal exceeds the ceiling tier', () => {
const out = spawnSync('git', ['ls-files', 'test/*.test.ts'], { cwd: ROOT, encoding: 'utf-8' });
const files = out.stdout.split('\n').filter((f) => f && isPaidTestFile(f));
expect(files.length).toBeGreaterThan(50); // scan-rot guard
const offenders: string[] = [];
for (const rel of files) {
const source = fs.readFileSync(path.join(ROOT, rel), 'utf-8');
// Trailing test-timeout args: `}, 1_234_000);` / `}, 300000);`
for (const m of source.matchAll(/\}\s*,\s*(\d[\d_]*)\s*(?:\/\*[^*]*\*\/\s*)?\)/g)) {
const ms = Number(m[1].replaceAll('_', ''));
if (ms > PTY_LONG_MS * 1.25) offenders.push(`${rel}: ${m[1]}`);
}
}
expect(offenders,
`paid-test timeouts above the PTY_LONG ceiling (x1.25 slack) are fiction ` +
`against the ${DEFAULT_SHARD_TIMEOUT_MS / 1000}s shard wall — split the test instead:\n${offenders.join('\n')}`,
).toEqual([]);
});
});
+387
View File
@@ -0,0 +1,387 @@
/**
* The eval CLI family — scripts/eval-select.ts, eval-list.ts, eval-compare.ts,
* eval-summary.ts — the primary interface to eval results.
*
* Isolation mechanisms (each verified against the source, not assumed):
*
* - eval-list / eval-compare / eval-summary resolve their eval dir via
* getProjectEvalDir() (test/helpers/eval-store.ts), which probes the
* CWD-RELATIVE `.claude/skills/gstack/bin/gstack-slug` first, then
* `~/.claude/...` (~ = $HOME of the child). They do NOT honor
* GSTACK_EVAL_DIR (only EvalCollector does). So the real isolation
* mechanism is: cwd = a temp HOME containing a fake gstack-slug that
* prints `SLUG=<fixture>`, routing every read to
* $HOME/.gstack/projects/<fixture>/evals — fully hermetic, and it
* exercises the primary (project-scoped) dir resolution path.
* (test/eval-list-cli.test.ts already covers the legacy-fallback dir +
* --limit validation; this file deliberately does not duplicate that.)
*
* - eval-select has NO isolation mechanism for its git diff: ROOT is
* hardcoded to the repo containing the script (import.meta.dir/..), so
* the CLI is smoke-tested against this repo with `--base HEAD` using
* shape invariants that hold for any working-tree state, and the
* "global touchfile ⇒ run everything" behavior is tested through the
* pure, importable selectTests() the CLI is a thin wrapper over.
*/
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { runBin } from './helpers/run-bin';
import { selectTests, E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES, GLOBAL_TOUCHFILES } from './helpers/touchfiles';
const ROOT = path.resolve(import.meta.dir, '..');
const SCRIPT = (name: string) => path.join(ROOT, 'scripts', name);
const SLUG = 'eval-cli-fixture';
let tmpHome: string;
let evalDir: string;
beforeEach(() => {
tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-eval-family-'));
// Fake gstack-slug at the cwd-relative probe path so getProjectEvalDir()
// deterministically resolves the project-scoped dir under the temp HOME.
const slugBin = path.join(tmpHome, '.claude', 'skills', 'gstack', 'bin');
fs.mkdirSync(slugBin, { recursive: true });
fs.writeFileSync(path.join(slugBin, 'gstack-slug'), `#!/usr/bin/env bash\necho "SLUG=${SLUG}"\n`, { mode: 0o755 });
evalDir = path.join(tmpHome, '.gstack', 'projects', SLUG, 'evals');
fs.mkdirSync(evalDir, { recursive: true });
});
afterEach(() => {
fs.rmSync(tmpHome, { recursive: true, force: true });
});
function runEvalCli(script: string, ...args: string[]) {
return runBin('bun', [SCRIPT(script), ...args], {
cwd: tmpHome,
home: tmpHome,
gstackHome: path.join(tmpHome, '.gstack'),
});
}
interface FixtureTest {
name: string;
passed: boolean;
cost?: number;
turns?: number;
duration?: number;
}
/** Write a run file in the collector's shapes: finalized `{version}-{branch}-{tier}-{ts}.json` or `_partial-e2e.json`. */
function writeRun(dir: string, opts: {
version?: string;
branch?: string;
tier?: 'e2e' | 'llm-judge';
timestamp: string;
tests: FixtureTest[];
partial?: boolean;
}): string {
const version = opts.version ?? '1.0.0';
const branch = opts.branch ?? 'featx';
const tier = opts.tier ?? 'e2e';
const tests = opts.tests.map(t => ({
name: t.name,
suite: 'fixture',
tier,
passed: t.passed,
duration_ms: t.duration ?? 1000,
cost_usd: t.cost ?? 0.5,
turns_used: t.turns ?? 5,
}));
const body = {
schema_version: 1,
version,
branch,
git_sha: 'abc1234',
timestamp: opts.timestamp,
hostname: 'fixture-host',
tier,
total_tests: tests.length,
passed: tests.filter(t => t.passed).length,
failed: tests.filter(t => !t.passed).length,
total_cost_usd: tests.reduce((s, t) => s + t.cost_usd, 0),
total_duration_ms: tests.reduce((s, t) => s + t.duration_ms, 0),
tests,
...(opts.partial ? { _partial: true } : {}),
};
const dateStr = opts.timestamp.replace(/[:.]/g, '').replace('T', '-').slice(0, 15);
const filename = opts.partial ? '_partial-e2e.json' : `${version}-${branch}-${tier}-${dateStr}.json`;
fs.mkdirSync(dir, { recursive: true });
const filepath = path.join(dir, filename);
fs.writeFileSync(filepath, JSON.stringify(body, null, 2) + '\n');
return filepath;
}
// ── eval-select ──────────────────────────────────────────────────────────────
describe('eval:select CLI (scripts/eval-select.ts)', () => {
test('--json parses and its selection partitions the full touchfile maps', () => {
// --base HEAD makes the committed diff empty; uncommitted/untracked files
// in the working tree may still appear, so assert shape invariants that
// hold for ANY tree state rather than pinning specific selections.
const result = runBin('bun', [SCRIPT('eval-select.ts'), '--json', '--base', 'HEAD'], { cwd: ROOT });
expect(result.status).toBe(0);
const parsed = JSON.parse(result.stdout);
expect(parsed.base).toBe('HEAD');
if (parsed.changed_files === 0) {
// Pristine tree: the no-diff shape reports run-all for both tiers.
expect(parsed.e2e).toBe('all');
expect(parsed.llm_judge).toBe('all');
expect(parsed.reason).toContain('all tests');
} else {
expect(Array.isArray(parsed.changed_files)).toBe(true);
expect(parsed.changed_files.length).toBeGreaterThan(0);
for (const [selection, map] of [
[parsed.e2e, E2E_TOUCHFILES],
[parsed.llm_judge, LLM_JUDGE_TOUCHFILES],
] as const) {
const total = Object.keys(map).length;
expect(Array.isArray(selection.selected)).toBe(true);
expect(Array.isArray(selection.skipped)).toBe(true);
// selected + skipped always partition the map: disjoint, complete.
expect(selection.selected.length + selection.skipped.length).toBe(total);
const overlap = selection.selected.filter((name: string) => selection.skipped.includes(name));
expect(overlap).toEqual([]);
expect(typeof selection.reason).toBe('string');
expect(selection.count).toBe(`${selection.selected.length}/${total}`);
}
expect(Array.isArray(parsed.e2e.removed_tests)).toBe(true);
}
});
test('human-readable mode prints the base and per-tier headers', () => {
const result = runBin('bun', [SCRIPT('eval-select.ts'), '--base', 'HEAD'], { cwd: ROOT });
expect(result.status).toBe(0);
expect(result.stdout).toContain('Base: HEAD');
// Either the no-diff line or the two selection headers.
const hasNoDiff = result.stdout.includes('No changed files detected');
if (!hasNoDiff) {
expect(result.stdout).toContain('E2E: selected');
expect(result.stdout).toContain('LLM-judge: selected');
}
});
test('a global-touchfile diff selects ALL tests with a global reason (pure selectTests)', () => {
// eval-select is a thin wrapper over selectTests(); the CLI cannot be
// pointed at a fixture repo (ROOT is hardcoded), so the run-all-on-global
// behavior is pinned through the same imported function it calls.
expect(GLOBAL_TOUCHFILES).toContain('test/helpers/eval-store.ts');
const selection = selectTests(['test/helpers/eval-store.ts'], E2E_TOUCHFILES, GLOBAL_TOUCHFILES);
expect(selection.reason).toBe('global: test/helpers/eval-store.ts');
expect(selection.selected.sort()).toEqual(Object.keys(E2E_TOUCHFILES).sort());
expect(selection.skipped).toEqual([]);
});
test('a per-test touchfile diff selects only the dependent test', () => {
const touchfiles = {
'test-a': ['src/feature-a.ts', 'src/shared/**'],
'test-b': ['src/feature-b.ts'],
};
const globals = ['helpers/global-runner.ts'];
const hitA = selectTests(['src/feature-a.ts'], touchfiles, globals);
expect(hitA.selected).toEqual(['test-a']);
expect(hitA.skipped).toEqual(['test-b']);
expect(hitA.reason).toBe('diff');
const hitGlob = selectTests(['src/shared/deep/util.ts'], touchfiles, globals);
expect(hitGlob.selected).toEqual(['test-a']);
const miss = selectTests(['docs/README.md'], touchfiles, globals);
expect(miss.selected).toEqual([]);
expect(miss.skipped.sort()).toEqual(['test-a', 'test-b']);
});
});
// ── eval-list ────────────────────────────────────────────────────────────────
describe('eval:list CLI (scripts/eval-list.ts)', () => {
test('empty eval dir prints the getting-started hint and exits 0', () => {
const result = runEvalCli('eval-list.ts');
expect(result.status).toBe(0);
expect(result.stdout).toContain('No eval runs yet');
});
test('lists finalized runs from the flat dir AND one level of shards/<slug>/', () => {
writeRun(evalDir, { branch: 'flat-branch', timestamp: '2026-01-01T01:00:00Z', tests: [{ name: 't1', passed: true, cost: 1.5, turns: 7 }] });
writeRun(path.join(evalDir, 'shards', 'shard-a'), { branch: 'shard-branch', timestamp: '2026-01-02T01:00:00Z', tests: [{ name: 't2', passed: true, cost: 0.5, turns: 3 }] });
const result = runEvalCli('eval-list.ts');
expect(result.status).toBe(0);
expect(result.stdout).toContain('Eval History (2 total runs)');
expect(result.stdout).toContain('flat-branch');
expect(result.stdout).toContain('shard-branch');
// Sorted by timestamp descending: the shard run (newer) is listed first.
expect(result.stdout.indexOf('shard-branch')).toBeLessThan(result.stdout.indexOf('flat-branch'));
// Reads route to the project-scoped dir resolved via the fake gstack-slug.
expect(result.stdout).toContain(path.join('projects', SLUG, 'evals'));
});
test('--branch and --tier filter the listing', () => {
writeRun(evalDir, { branch: 'keep-me', tier: 'e2e', timestamp: '2026-01-01T01:00:00Z', tests: [{ name: 't1', passed: true }] });
writeRun(evalDir, { branch: 'drop-me', tier: 'llm-judge', timestamp: '2026-01-02T01:00:00Z', tests: [{ name: 't2', passed: true }] });
const byBranch = runEvalCli('eval-list.ts', '--branch', 'keep-me');
expect(byBranch.status).toBe(0);
expect(byBranch.stdout).toContain('Eval History (1 total runs)');
expect(byBranch.stdout).toContain('keep-me');
expect(byBranch.stdout).not.toContain('drop-me');
const byTier = runEvalCli('eval-list.ts', '--tier', 'llm-judge');
expect(byTier.status).toBe(0);
expect(byTier.stdout).toContain('drop-me');
expect(byTier.stdout).not.toContain('keep-me');
});
test('DOCUMENTS CURRENT BEHAVIOR: in-progress _partial accumulators appear in the listing', () => {
// eval-list.ts applies NO isPartialEval filter (unlike eval-compare and
// every baseline lookup in eval-store.ts), so the in-progress accumulator
// is listed as if it were a run. If eval-list ever grows a partial filter,
// update this test to assert exclusion — that would be an improvement,
// not a regression.
writeRun(evalDir, { branch: 'finalized-run', timestamp: '2026-01-01T01:00:00Z', tests: [{ name: 't1', passed: true }] });
writeRun(evalDir, { branch: 'partial-sentinel', timestamp: '2026-01-03T01:00:00Z', tests: [{ name: 't1', passed: false }], partial: true });
const result = runEvalCli('eval-list.ts');
expect(result.status).toBe(0);
expect(result.stdout).toContain('finalized-run');
expect(result.stdout).toContain('Eval History (2 total runs)');
expect(result.stdout).toContain('partial-sentinel');
});
});
// ── eval-compare ─────────────────────────────────────────────────────────────
describe('eval:compare CLI (scripts/eval-compare.ts)', () => {
test('empty eval dir prints the getting-started hint and exits 0', () => {
const result = runEvalCli('eval-compare.ts');
expect(result.status).toBe(0);
expect(result.stdout).toContain('No eval runs yet');
});
test('a single run is not enough to compare (exit 0 with guidance)', () => {
writeRun(evalDir, { timestamp: '2026-01-01T01:00:00Z', tests: [{ name: 't1', passed: true }] });
const result = runEvalCli('eval-compare.ts');
expect(result.status).toBe(0);
expect(result.stdout).toContain('Need at least 2 eval runs');
});
test('no args: compares the two most recent FINALIZED runs and reports deltas; the fresher partial is never a side', () => {
writeRun(evalDir, {
timestamp: '2026-01-01T01:00:00Z',
tests: [
{ name: 't-stable', passed: true, cost: 1.0, turns: 5 },
{ name: 't-flaky', passed: false, cost: 1.0, turns: 5 },
{ name: 't-regressed', passed: true, cost: 1.0, turns: 5 },
],
});
writeRun(evalDir, {
timestamp: '2026-01-02T01:00:00Z',
tests: [
{ name: 't-stable', passed: true, cost: 1.0, turns: 5 },
{ name: 't-flaky', passed: true, cost: 1.0, turns: 5 },
{ name: 't-regressed', passed: false, cost: 1.0, turns: 5 },
],
});
// Freshest timestamp of all — if partials leaked into selection, this
// would be picked as the "after" run (or the baseline) and its sentinel
// branch would show up in the header line.
writeRun(evalDir, {
branch: 'partial-sentinel',
timestamp: '2026-01-03T01:00:00Z',
tests: [{ name: 't-stable', passed: false }],
partial: true,
});
const result = runEvalCli('eval-compare.ts');
expect(result.status).toBe(0);
expect(result.stdout).not.toContain('partial-sentinel');
expect(result.stdout).toContain('1 improved');
expect(result.stdout).toContain('1 regressed');
expect(result.stdout).toContain('1 unchanged');
expect(result.stdout).toContain('REGRESSION: "t-regressed" was passing, now fails.');
expect(result.stdout).toContain('Fixed: "t-flaky" now passes.');
});
test('two explicit filenames resolve relative to the eval dir and compare in the given order', () => {
const before = writeRun(evalDir, {
timestamp: '2026-01-01T01:00:00Z',
tests: [{ name: 't-x', passed: true, cost: 1.0 }],
});
const after = writeRun(evalDir, {
timestamp: '2026-01-02T01:00:00Z',
tests: [{ name: 't-x', passed: false, cost: 3.0 }],
});
const result = runEvalCli('eval-compare.ts', path.basename(before), path.basename(after));
expect(result.status).toBe(0);
expect(result.stdout).toContain('1 regressed');
expect(result.stdout).toContain('REGRESSION: "t-x" was passing, now fails.');
// Cost delta: 1.00 → 3.00 = +$2.00
expect(result.stdout).toContain('+$2.00');
});
test('a missing explicit file fails with exit 1 and names the resolved path', () => {
writeRun(evalDir, { timestamp: '2026-01-01T01:00:00Z', tests: [{ name: 't1', passed: true }] });
writeRun(evalDir, { timestamp: '2026-01-02T01:00:00Z', tests: [{ name: 't1', passed: true }] });
const result = runEvalCli('eval-compare.ts', 'does-not-exist.json', 'also-missing.json');
expect(result.status).toBe(1);
expect(result.stderr).toContain('File not found:');
expect(result.stderr).toContain('does-not-exist.json');
});
});
// ── eval-summary ─────────────────────────────────────────────────────────────
describe('eval:summary CLI (scripts/eval-summary.ts)', () => {
test('empty eval dir prints the getting-started hint and exits 0', () => {
const result = runEvalCli('eval-summary.ts');
expect(result.status).toBe(0);
expect(result.stdout).toContain('No eval runs yet');
});
test('aggregates run counts, spend, and flaky tests across tiers', () => {
writeRun(evalDir, {
tier: 'e2e',
branch: 'branch-one',
timestamp: '2026-01-01T01:00:00Z',
tests: [
{ name: 't-flaky', passed: true, cost: 0.5, turns: 4, duration: 10_000 },
{ name: 't-solid', passed: true, cost: 0.5, turns: 6, duration: 20_000 },
],
});
writeRun(evalDir, {
tier: 'e2e',
branch: 'branch-one',
timestamp: '2026-01-02T01:00:00Z',
tests: [
{ name: 't-flaky', passed: false, cost: 1.0, turns: 8, duration: 30_000 },
{ name: 't-solid', passed: true, cost: 1.0, turns: 6, duration: 20_000 },
],
});
writeRun(evalDir, {
tier: 'llm-judge',
branch: 'branch-two',
timestamp: '2026-01-03T01:00:00Z',
tests: [{ name: 'judge-1', passed: true, cost: 0.5 }],
});
const result = runEvalCli('eval-summary.ts');
expect(result.status).toBe(0);
// 3 runs total: 2 e2e + 1 llm-judge.
expect(result.stdout).toContain('3 (2 e2e, 1 llm-judge)');
// Total spend: (0.5+0.5) + (1.0+1.0) + 0.5 = 3.50
expect(result.stdout).toContain('$3.50');
// t-flaky passed once and failed once → flagged flaky, keyed by tier.
expect(result.stdout).toContain('Flaky tests (1):');
expect(result.stdout).toContain('e2e:t-flaky');
expect(result.stdout).not.toContain('e2e:t-solid');
// Date range spans first → last timestamp.
expect(result.stdout).toContain('2026-01-01 01:00');
expect(result.stdout).toContain('2026-01-03 01:00');
expect(result.stdout).toContain(path.join('projects', SLUG, 'evals'));
});
});
+18 -17
View File
@@ -48,28 +48,29 @@ const KNOWN_MATRIX_GAPS = new Set([
'test/skill-e2e-plan-design-with-ui.test.ts',
'test/skill-e2e-plan-devex-finding-floor.test.ts',
'test/skill-e2e-plan-devex-plan-mode.test.ts',
// Exposed by the 2026-08 dep-list self-registration sweep: these eight had
// zero gate-key dep-list membership before it, so the census never saw
// them as gate-hosting. Their gate tests run in NO CI lane today. The
// paid-lane re-platform (test-paid-shards.ts as the CI engine) runs every
// gate-tier file by construction and retires this whole ratchet.
'test/skill-e2e-cso.test.ts',
'test/skill-e2e-diagram.test.ts',
'test/skill-e2e-learnings.test.ts',
'test/skill-e2e-plan-tune.test.ts',
'test/skill-e2e-plan-tune-cathedral.test.ts',
'test/skill-e2e-review-army.test.ts',
'test/skill-e2e-session-intelligence.test.ts',
'test/skill-e2e-skillify.test.ts',
]);
/**
* Matrix files whose whole-file tier guard has no matching row `tier:`
* property (pre-existing, found 2026-08-26). Consequences today:
* - codex-e2e / gemini-e2e declare 'periodic' → both jobs run ZERO tests and
* report green on every PR (vestigial rows; the periodic cron lane owns
* these suites).
* - the two PTY plan-mode smokes declare 'gate' → the e2e-pty-plan-smoke job
* spends ~7 min on container setup and skill registration, then bun test
* skips every describe — hollow-green since the files adopted
* describeE2ETier.
* Fixing either means deliberately (re)activating paid suites on every PR —
* tracked in the same TODOS burn-down. Fix = add `tier:` to the row (or
* delete the vestigial row), then DELETE the entry here.
* property. Burned down to empty 2026-08-29: the vestigial codex/gemini rows
* were deleted (periodic-tier files, zero tests per PR) and
* e2e-pty-plan-smoke gained its `tier: gate`. The ratchet stays so a future
* row/file tier mismatch fails the suite instead of shipping hollow green.
*/
const KNOWN_TIER_UNSET = new Map([
['test/codex-e2e.test.ts', 'periodic'],
['test/gemini-e2e.test.ts', 'periodic'],
['test/skill-e2e-office-hours-auto-mode.test.ts', 'gate'],
['test/skill-e2e-plan-mode-no-op.test.ts', 'gate'],
]);
const KNOWN_TIER_UNSET = new Map<string, string>([]);
interface MatrixRow {
name: string;
+3 -5
View File
@@ -11,20 +11,18 @@ let gstackHome: string;
let repoDir: string;
import { gitIn, findFilesBySuffix } from './helpers/scratch-repo';
import { runBin } from './helpers/run-bin';
function git(args: string) {
gitIn(repoDir, args);
}
function run(args: string[], opts: { cwd?: string } = {}): { status: number; stdout: string; stderr: string } {
const r = spawnSync(EVIDENCE, args, {
return runBin(EVIDENCE, args, {
cwd: opts.cwd ?? repoDir,
env: { ...process.env, GSTACK_HOME: gstackHome },
encoding: 'utf-8',
timeout: 60000,
env: { GSTACK_HOME: gstackHome },
maxBuffer: 16 * 1024 * 1024, // the truncation test streams 3MB through the wrapper
});
return { status: r.status ?? 1, stdout: r.stdout ?? '', stderr: r.stderr ?? '' };
}
function ledgerFile(): string {
+5 -14
View File
@@ -12,7 +12,8 @@ import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { spawnSync } from 'child_process';
import { runBin } from './helpers/run-bin';
const ROOT = path.resolve(import.meta.dir, '..');
const BIN_CONFIG = path.join(ROOT, 'bin', 'gstack-config');
@@ -28,19 +29,9 @@ afterEach(() => {
});
function run(...args: string[]): { stdout: string; stderr: string; status: number } {
// gstack-config precedence is `${GSTACK_HOME:-${GSTACK_STATE_DIR:-$HOME/.gstack}}`,
// so GSTACK_HOME from the developer's parent env wins over the test's
// GSTACK_STATE_DIR. Override both to isolate from the real ~/.gstack.
const res = spawnSync(BIN_CONFIG, args, {
env: { ...process.env, GSTACK_STATE_DIR: tmpHome, GSTACK_HOME: tmpHome },
encoding: 'utf-8',
cwd: ROOT,
});
return {
stdout: (res.stdout ?? '').trim(),
stderr: (res.stderr ?? '').trim(),
status: res.status ?? -1,
};
// runBin's gstackHome sets GSTACK_HOME + GSTACK_STATE_DIR together — the
// config-precedence isolation this file used to document by hand.
return runBin(BIN_CONFIG, args, { gstackHome: tmpHome, cwd: ROOT, trim: true });
}
describe('gstack-config explain_level', () => {
+18 -24
View File
@@ -9,7 +9,8 @@
* factory, opencode, openclaw, cursor, kiro).
*
* Tests drive gen-skill-docs as a subprocess against a temp GSTACK_HOME
* with each detection state, then assert what landed in the generated
* with each detection state, rendering into an isolated --out-dir (never
* writing the working tree), then assert what landed in the rendered
* Claude-host SKILL.md. This is end-to-end through the actual override
* pipeline — no mocking — so it catches regressions in either the loader
* or the suppressedResolvers filter.
@@ -18,9 +19,9 @@
* generation against the real repo; --host claude scopes to one host).
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { describe, test, expect } from 'bun:test';
import { execFileSync } from 'child_process';
import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs';
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
@@ -49,33 +50,29 @@ function makeFixture(detectionJson: string | null): FixtureEnv {
}
/**
* Run gen-skill-docs with --respect-detection and an isolated GSTACK_HOME.
* Returns the regenerated office-hours/SKILL.md content WITHOUT writing
* over the committed file: we use --dry-run to keep the working tree
* clean, then parse the output via re-reading the committed file... no,
* that doesn't work for dry-run since dry-run doesn't write.
*
* Approach: generate to a temp output dir by running gen-skill-docs in a
* temp checkout. Simpler alternative: actually regenerate, snapshot the
* file content, then git-checkout the committed version back. We use this
* since gen-skill-docs doesn't expose an output-path arg.
* Run gen-skill-docs with --respect-detection and an isolated GSTACK_HOME,
* rendering into a fresh --out-dir. The working tree is never written: the
* generator reads its inputs (templates, resolvers) from the repo but lands
* every output in the temp dir, which we snapshot and delete. This replaced
* the old mutate-then-restore approach (which regenerated the committed
* files in place and only restored the probe files, leaving every OTHER
* generated file rewritten — a partial-restore hazard for concurrent
* readers).
*/
function regenAndSnapshot(opts: {
respectDetection: boolean;
tmpHome: string;
files: string[];
}): Map<string, string> {
// Save committed content so we can restore after snapshotting.
const original = new Map<string, string>();
for (const f of opts.files) {
original.set(f, readFileSync(join(REPO_ROOT, f), 'utf-8'));
}
const outDir = mkdtempSync(join(tmpdir(), 'gbrain-detect-out-'));
const args = [
'run',
'scripts/gen-skill-docs.ts',
'--host',
'claude',
'--out-dir',
outDir,
];
if (opts.respectDetection) args.push('--respect-detection');
@@ -87,17 +84,14 @@ function regenAndSnapshot(opts: {
timeout: 30_000,
});
// Snapshot the regenerated content.
// Snapshot the rendered content from the out-dir.
const snapshot = new Map<string, string>();
for (const f of opts.files) {
snapshot.set(f, readFileSync(join(REPO_ROOT, f), 'utf-8'));
snapshot.set(f, readFileSync(join(outDir, f), 'utf-8'));
}
return snapshot;
} finally {
// Always restore so the test leaves the working tree clean.
for (const [f, content] of original) {
writeFileSync(join(REPO_ROOT, f), content);
}
rmSync(outDir, { recursive: true, force: true });
}
}
+3 -2
View File
@@ -15,6 +15,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS } from './helpers/eval-budgets';
import { runGeminiSkill } from './helpers/gemini-session-runner';
import type { GeminiResult } from './helpers/gemini-session-runner';
import { EvalCollector } from './helpers/eval-store';
@@ -151,7 +152,7 @@ describeGemini('Gemini E2E', () => {
// Uses a simple prompt that doesn't require skill invocation or complex navigation.
const result = await runGeminiSkill({
prompt: 'What is this project? Answer in one sentence based on the README.',
timeoutMs: 90_000,
timeoutMs: JUDGE_MS,
cwd: testWorktree,
});
@@ -163,5 +164,5 @@ describeGemini('Gemini E2E', () => {
recordGeminiE2E('gemini-smoke', result, passed);
expect(result.output.length, 'Gemini should produce output').toBeGreaterThan(10);
}, 120_000);
}, JUDGE_MS);
});
+79 -61
View File
@@ -12,18 +12,26 @@
* file's timestamp never matched the latest gen. Fixed in 43e18af4 — this
* test pins the contract going forward.
*
* The test pays a small cost (~2 gen-skill-docs invocations, ~3s total) but
* catches a class of bugs that's invisible until CI fails.
* Isolation: each run renders into its OWN --out-dir (the working tree is
* never written), and the two out-dirs are diffed RECURSIVELY byte-for-byte
* — strictly stronger than the old sampled-file snapshot of an in-place
* double regen. The only tolerated difference is the out-dir path itself:
* --out-dir repoints section-base paths into the render, so each file is
* normalized by replacing its own out-dir path with a placeholder before
* comparison. Any OTHER byte difference (timestamp, random ID, iteration
* order) still fails.
*/
import { describe, test, expect } from 'bun:test';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
const REPO_ROOT = path.resolve(import.meta.dir, '..');
/** Files that gen-skill-docs writes and that must be byte-stable across runs. */
/** Presence sanity list: key Claude-host outputs that must exist in a render
* (guards the recursive diff against vacuously comparing two empty dirs). */
const STABLE_OUTPUTS = [
'SKILL.md',
'ship/SKILL.md',
@@ -33,10 +41,11 @@ const STABLE_OUTPUTS = [
];
/**
* Sampled outputs from EVERY non-Claude host. The full host-all run touches
* .agents/, .cursor/, .factory/, .gbrain/, .hermes/, .kiro/, .openclaw/,
* .opencode/, .slate/ — picking one canonical file per host catches per-host
* non-determinism without paying the cost of snapshotting hundreds of files.
* Presence sanity for the --host all render: one canonical file per
* representative non-Claude host. The full host-all run touches .agents/,
* .cursor/, .factory/, .gbrain/, .hermes/, .kiro/, .openclaw/, .opencode/,
* .slate/ — the recursive diff covers every file; this list only proves the
* render actually fanned out across hosts.
*/
const STABLE_HOST_ALL_OUTPUTS = [
'SKILL.md',
@@ -59,51 +68,83 @@ function runGen(extraArgs: string[] = []): { exitCode: number; stderr: string }
};
}
function snapshot(files: string[] = STABLE_OUTPUTS): Map<string, string> {
const m = new Map<string, string>();
for (const rel of files) {
const full = path.join(REPO_ROOT, rel);
if (fs.existsSync(full)) {
m.set(rel, fs.readFileSync(full, 'utf-8'));
}
/** Recursively list all regular files under dir as sorted relative paths. */
function listFiles(dir: string, prefix = ''): string[] {
const out: string[] = [];
for (const entry of fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
const rel = prefix ? `${prefix}/${entry.name}` : entry.name;
if (entry.isDirectory()) out.push(...listFiles(path.join(dir, entry.name), rel));
else out.push(rel);
}
return m;
return out;
}
describe('gen-skill-docs idempotency', () => {
test('two consecutive runs produce byte-identical outputs (no flapping fields)', () => {
const firstRun = runGen();
/**
* Diff two render dirs recursively. Every generated output is text, so files
* are read as utf-8 and each dir's own absolute path is normalized to
* <OUT_DIR> (the section-base repoint is the ONLY sanctioned difference
* between two renders of the same tree). Returns human-readable mismatches.
*/
function diffRenderDirs(dirA: string, dirB: string): string[] {
const filesA = listFiles(dirA);
const filesB = listFiles(dirB);
const problems: string[] = [];
const setB = new Set(filesB);
for (const f of filesA) {
if (!setB.has(f)) { problems.push(`${f} (only in first render)`); continue; }
const a = fs.readFileSync(path.join(dirA, f), 'utf-8').replaceAll(dirA, '<OUT_DIR>');
const b = fs.readFileSync(path.join(dirB, f), 'utf-8').replaceAll(dirB, '<OUT_DIR>');
if (a !== b) problems.push(`${f} (content differs)`);
}
const setA = new Set(filesA);
for (const f of filesB) {
if (!setA.has(f)) problems.push(`${f} (only in second render)`);
}
return problems;
}
/** Render twice into two fresh out-dirs, assert byte-identical outputs. */
function assertDoubleRenderStable(extraArgs: string[], presenceSanity: string[], label: string): void {
const outA = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-idem-a-'));
const outB = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-idem-b-'));
try {
const firstRun = runGen([...extraArgs, '--out-dir', outA]);
expect(firstRun.exitCode).toBe(0);
const after1 = snapshot();
expect(after1.size).toBeGreaterThan(0);
const secondRun = runGen();
const secondRun = runGen([...extraArgs, '--out-dir', outB]);
expect(secondRun.exitCode).toBe(0);
const after2 = snapshot();
// Compare each stable output byte-for-byte.
const flapping: string[] = [];
for (const [file, before] of after1.entries()) {
const now = after2.get(file);
if (now !== before) flapping.push(file);
// Non-vacuous guard: the key outputs actually rendered.
for (const rel of presenceSanity) {
expect({ file: rel, exists: fs.existsSync(path.join(outA, rel)) })
.toEqual({ file: rel, exists: true });
}
const flapping = diffRenderDirs(outA, outB);
if (flapping.length > 0) {
throw new Error(
`${flapping.length} file(s) changed between two consecutive gen-skill-docs runs (flapping):\n` +
`${flapping.length} file(s) differ between two consecutive ${label} gen runs (flapping):\n` +
flapping.map(f => ` - ${f}`).join('\n') +
`\nLikely cause: a non-deterministic field (timestamp, random ID, ` +
`filesystem-iteration order) leaked into the generated output. CI freshness ` +
`checks (git diff --exit-code) will fail unpredictably until this is fixed.`,
);
}
} finally {
fs.rmSync(outA, { recursive: true, force: true });
fs.rmSync(outB, { recursive: true, force: true });
}
}
describe('gen-skill-docs idempotency', () => {
test('two consecutive runs produce byte-identical outputs (no flapping fields)', () => {
assertDoubleRenderStable([], STABLE_OUTPUTS, 'claude-host');
}, 180_000); // ~2 min budget for two gen runs
test('--dry-run after a fresh gen reports zero stale files', () => {
// Pre-condition: working tree gen must be fresh (idempotency test above ran first).
// If a contributor introduces a non-deterministic field, this dry-run reports STALE.
test('--dry-run against the tracked tree reports zero stale files', () => {
// Tracked-tree freshness assertion (deliberately a READ of the committed
// files — the out-dir renders above never touch them). If a contributor
// edits a template without regenerating, or introduces a
// non-deterministic field, this dry-run reports STALE.
const result = spawnSync('bun', ['run', 'gen:skill-docs', '--dry-run'], {
cwd: REPO_ROOT,
stdio: ['ignore', 'pipe', 'pipe'],
@@ -115,7 +156,7 @@ describe('gen-skill-docs idempotency', () => {
const staleLines = stdout.split('\n').filter(l => l.startsWith('STALE:'));
if (staleLines.length > 0) {
throw new Error(
`--dry-run reports ${staleLines.length} stale file(s) after a fresh gen:\n` +
`--dry-run reports ${staleLines.length} stale file(s) against the tracked tree:\n` +
staleLines.map(l => ` ${l}`).join('\n') +
`\nRun \`bun run gen:skill-docs\` and commit the result.`,
);
@@ -127,31 +168,8 @@ describe('gen-skill-docs idempotency', () => {
// (Codex, Factory, Cursor, OpenClaw, GBrain, Slate, OpenCode, Hermes,
// Kiro) have their own output paths and could carry their own
// non-deterministic fields. We hit a "--host all needed for freshness
// check" mid-/ship; this test pins the contract across every host.
const firstRun = runGen(['--host', 'all']);
expect(firstRun.exitCode).toBe(0);
const after1 = snapshot(STABLE_HOST_ALL_OUTPUTS);
expect(after1.size).toBeGreaterThan(0);
const secondRun = runGen(['--host', 'all']);
expect(secondRun.exitCode).toBe(0);
const after2 = snapshot(STABLE_HOST_ALL_OUTPUTS);
const flapping: string[] = [];
for (const [file, before] of after1.entries()) {
const now = after2.get(file);
if (now !== before) flapping.push(file);
}
if (flapping.length > 0) {
throw new Error(
`${flapping.length} file(s) changed between two consecutive --host all gen runs:\n` +
flapping.map(f => ` - ${f}`).join('\n') +
`\nLikely cause: a non-deterministic field leaked into a non-Claude host's ` +
`config or resolver output. CI freshness checks for that host will flap.`,
);
}
// check" mid-/ship; this test pins the contract across every host — the
// recursive diff covers EVERY rendered file for EVERY host.
assertDoubleRenderStable(['--host', 'all'], STABLE_HOST_ALL_OUTPUTS, '--host all');
}, 300_000); // ~5 min budget for two host-all runs
});
+52
View File
@@ -0,0 +1,52 @@
/**
* Importing scripts/gen-skill-docs.ts must not touch the tree.
*
* Before the main() guard, the generator's whole body executed at module
* load: any `import`/`require` of it (test/gen-skill-docs.test.ts pulls
* assertSinglePreamble; test/catalog-trim.test.ts imports helpers)
* regenerated all 71 SKILL.md in place — the root cause of half the
* TREE_MUTATING serial-shard entries (hazard class #2532). A regression
* here silently re-poisons parallel shards with mid-window tree rewrites.
*
* The probe runs in a subprocess so a regression can't contaminate THIS
* process, and asserts on mtimes rather than git status — the working tree
* may legitimately carry uncommitted SKILL.md edits while this runs; what
* must not happen is the import WRITING files.
*/
import { describe, expect, test } from 'bun:test';
import * as path from 'node:path';
const ROOT = path.resolve(__dirname, '..');
describe('gen-skill-docs import purity', () => {
test('importing the module neither writes SKILL.md nor runs main()', () => {
const probe = `
const fs = require('node:fs');
const path = require('node:path');
const ROOT = ${JSON.stringify(ROOT)};
const targets = [
path.join(ROOT, 'ship', 'SKILL.md'),
path.join(ROOT, 'review', 'SKILL.md'),
path.join(ROOT, 'gstack', 'llms.txt'),
].filter((p) => fs.existsSync(p));
if (targets.length === 0) throw new Error('probe rot: no generated targets found');
const before = targets.map((p) => fs.statSync(p).mtimeMs);
const mod = require(path.join(ROOT, 'scripts', 'gen-skill-docs.ts'));
if (typeof mod.main !== 'function') throw new Error('main() export missing');
const after = targets.map((p) => fs.statSync(p).mtimeMs);
for (let i = 0; i < targets.length; i++) {
if (before[i] !== after[i]) throw new Error('import mutated ' + targets[i]);
}
console.log('IMPORT_PURE');
`;
const out = Bun.spawnSync(['bun', '-e', probe], { cwd: ROOT });
const stdout = out.stdout.toString();
const stderr = out.stderr.toString();
expect(stderr, stderr).not.toContain('import mutated');
expect(stdout).toContain('IMPORT_PURE');
// The import must also not have run generation output (the "GENERATED:"
// lines main() prints) — load-time execution is the exact regression.
expect(stdout).not.toContain('GENERATED:');
expect(out.exitCode).toBe(0);
});
});
+73
View File
@@ -94,4 +94,77 @@ describe('gen-skill-docs --out-dir (B2 render isolation)', () => {
fs.rmSync(outDir, { recursive: true, force: true });
}
});
// ── External-host out-dir cases ─────────────────────────────
// The former tree-mutating tests read codex/factory artifacts from out-dir
// renders. That is only sound if an out-dir external render is (a) clean —
// zero tracked-tree dirt — and (b) byte-identical to what the in-place
// render would have produced. Both halves are pinned here.
test('--host codex --out-dir adds no tracked dirt and is byte-identical to the in-place render', () => {
const outDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-out-codex-'));
const inPlaceShip = path.join(ROOT, '.agents', 'skills', 'gstack-ship', 'SKILL.md');
// Compared before/after rather than asserting empty, so a dev's own
// unrelated dirty files can't false-fail the suite (#2569 pattern).
const beforePorcelain = porcelain();
try {
// 1) Fresh IN-PLACE codex render — the existing behavior: it writes
// only the gitignored .agents/ tree (itself invisible to porcelain).
const inPlace = spawnSync(
'bun',
['run', 'scripts/gen-skill-docs.ts', '--host', 'codex'],
{ cwd: ROOT, encoding: 'utf-8', timeout: 120_000 },
);
expect(inPlace.status).toBe(0);
expect(porcelain()).toBe(beforePorcelain);
const inPlaceBytes = fs.readFileSync(inPlaceShip);
// 2) Out-dir render: zero new dirt, same bytes.
const res = spawnSync(
'bun',
['run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--out-dir', outDir],
{ cwd: ROOT, encoding: 'utf-8', timeout: 120_000 },
);
expect(res.status).toBe(0);
expect(porcelain()).toBe(beforePorcelain);
const outShip = path.join(outDir, '.agents', 'skills', 'gstack-ship', 'SKILL.md');
expect(fs.existsSync(outShip)).toBe(true);
expect(fs.readFileSync(outShip).equals(inPlaceBytes)).toBe(true);
// Codex metadata (agents/openai.yaml) mirrors into the out-dir too.
expect(fs.existsSync(path.join(outDir, '.agents', 'skills', 'gstack-ship', 'agents', 'openai.yaml'))).toBe(true);
} finally {
fs.rmSync(outDir, { recursive: true, force: true });
}
}, 120_000);
test('--host all --out-dir renders every host tree into the out-dir; tracked tree stays clean', () => {
const outDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-out-all-'));
const beforePorcelain = porcelain();
try {
const res = spawnSync(
'bun',
['run', 'scripts/gen-skill-docs.ts', '--host', 'all', '--out-dir', outDir],
{ cwd: ROOT, encoding: 'utf-8', timeout: 300_000 },
);
expect(res.status).toBe(0);
// Zero new dirt in the source checkout.
expect(porcelain()).toBe(beforePorcelain);
// Claude host + external hosts + openclaw docs + llms.txt all landed in the out-dir.
for (const rel of [
'ship/SKILL.md',
'.agents/skills/gstack-ship/SKILL.md',
'.factory/skills/gstack-ship/SKILL.md',
'gstack/llms.txt',
'openclaw/gstack-lite-CLAUDE.md',
]) {
expect({ file: rel, exists: fs.existsSync(path.join(outDir, rel)) })
.toEqual({ file: rel, exists: true });
}
} finally {
fs.rmSync(outDir, { recursive: true, force: true });
}
}, 300_000);
});
+97 -109
View File
@@ -1,4 +1,4 @@
import { describe, test, expect, beforeAll } from 'bun:test';
import { describe, test, expect, afterAll } from 'bun:test';
import { assertSinglePreamble } from '../scripts/gen-skill-docs';
import { COMMAND_DESCRIPTIONS } from '../browse/src/commands';
import { SNAPSHOT_FLAGS } from '../browse/src/snapshot';
@@ -125,6 +125,32 @@ import { getHostConfig as __getHostConfig } from '../hosts/index';
const CLAUDE_SKIPPED = new Set(__getHostConfig('claude').generation.skipSkills ?? []);
const CLAUDE_GENERATED_SKILLS = ALL_SKILLS.filter(s => !CLAUDE_SKIPPED.has(s.dir));
// ─── Out-dir render isolation ────────────────────────────────
// Every generator invocation in this file that used to regenerate the live
// tree (the gitignored .agents/.factory/... host dirs included) now renders
// into this module-level out-dir: ONE `--host all` render covers the claude
// host plus every external host, and all golden-artifact reads plus the
// per-host `--dry-run` determinism checks point here. The tracked tree is
// only ever READ (the `generated files are fresh` dry-run deliberately
// compares against the committed files — that is a read, not a write).
// Out-dir renders of external hosts are byte-identical to in-place renders
// (pinned by test/gen-skill-docs-out-dir.test.ts).
const EXTERNAL_OUT = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-gen-docs-out-'));
{
const render = Bun.spawnSync(
['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'all', '--out-dir', EXTERNAL_OUT],
{ cwd: ROOT, stdout: 'pipe', stderr: 'pipe' },
);
if (render.exitCode !== 0) {
throw new Error(
`gen-skill-docs --host all --out-dir failed (exit ${render.exitCode}):\n${render.stderr.toString()}`,
);
}
}
afterAll(() => {
fs.rmSync(EXTERNAL_OUT, { recursive: true, force: true });
});
describe('gen-skill-docs', () => {
// Browse carve (token-reduction Phase 4): the command reference + snapshot
// flags render into browse/sections/command-list.md now — read the
@@ -219,8 +245,9 @@ describe('gen-skill-docs', () => {
});
test('every generated Codex (.agents/skills) frontmatter parses as strict YAML', () => {
const agentsDir = path.join(ROOT, '.agents', 'skills');
if (!fs.existsSync(agentsDir)) return; // skip if external hosts not generated
// Reads the module-level out-dir render (guaranteed present — the render
// throws at module load if it fails), never the live gitignored tree.
const agentsDir = path.join(EXTERNAL_OUT, '.agents', 'skills');
for (const entry of fs.readdirSync(agentsDir, { withFileTypes: true })) {
if (!entry.isDirectory()) continue;
const mdPath = path.join(agentsDir, entry.name, 'SKILL.md');
@@ -240,8 +267,7 @@ describe('gen-skill-docs', () => {
});
test(`every Codex SKILL.md description stays within ${MAX_SKILL_DESCRIPTION_LENGTH} chars`, () => {
const agentsDir = path.join(ROOT, '.agents', 'skills');
if (!fs.existsSync(agentsDir)) return; // skip if not generated
const agentsDir = path.join(EXTERNAL_OUT, '.agents', 'skills');
for (const entry of fs.readdirSync(agentsDir, { withFileTypes: true })) {
if (!entry.isDirectory()) continue;
const skillMd = path.join(agentsDir, entry.name, 'SKILL.md');
@@ -254,8 +280,7 @@ describe('gen-skill-docs', () => {
test('every Codex SKILL.md description stays under 900-char warning threshold', () => {
const WARN_THRESHOLD = 900;
const agentsDir = path.join(ROOT, '.agents', 'skills');
if (!fs.existsSync(agentsDir)) return;
const agentsDir = path.join(EXTERNAL_OUT, '.agents', 'skills');
const violations: string[] = [];
for (const entry of fs.readdirSync(agentsDir, { withFileTypes: true })) {
if (!entry.isDirectory()) continue;
@@ -283,6 +308,9 @@ describe('gen-skill-docs', () => {
});
test('generated files are fresh (match --dry-run)', () => {
// Deliberately compares against the LIVE TRACKED SKILL.md files (no
// --out-dir): this is the freshness gate for the committed tree. Dry-run
// writes nothing — it is a read.
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--dry-run'], {
cwd: ROOT,
stdout: 'pipe',
@@ -1373,7 +1401,7 @@ describe('DESIGN_SKETCH resolver', () => {
describe('CODEX_SECOND_OPINION resolver', () => {
const content = readSkillUnion('office-hours'); // carved: Phase 5/6 prose moved to section
const codexContent = fs.readFileSync(path.join(ROOT, '.agents', 'skills', 'gstack-office-hours', 'SKILL.md'), 'utf-8');
const codexContent = fs.readFileSync(path.join(EXTERNAL_OUT, '.agents', 'skills', 'gstack-office-hours', 'SKILL.md'), 'utf-8');
test('Phase 3.5 section appears in office-hours SKILL.md', () => {
expect(content).toContain('Phase 3.5: Cross-Model Second Opinion');
@@ -1810,35 +1838,24 @@ describe('DESIGN_REVIEW_LITE extended with Codex', () => {
// ─── Codex Generation Tests ─────────────────────────────────
describe('Codex generation (--host codex)', () => {
const AGENTS_DIR = path.join(ROOT, '.agents', 'skills');
// .agents/ is gitignored (v0.11.2.0) — read the module-level out-dir render
// (--host all covers codex) instead of regenerating the live tree in place.
const AGENTS_DIR = path.join(EXTERNAL_OUT, '.agents', 'skills');
// .agents/ is gitignored (v0.11.2.0) — generate on demand for tests
Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'codex'], {
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
});
// Dynamic discovery of expected Codex skills: all templates except /codex
// Also excludes skills where .agents/skills/{name} is a symlink back to the repo root
// (vendored dev mode — gen-skill-docs skips these to avoid overwriting Claude SKILL.md)
// Dynamic discovery of expected Codex skills: all templates except /codex.
// The out-dir is a fresh mkdtemp, so the vendored-dev-mode symlink loop
// (.agents/skills/{name} → repo root) that made the generator skip skills
// in-place can never occur here — every template renders.
const CODEX_SKILLS = (() => {
const skills: Array<{ dir: string; codexName: string }> = [];
const isSymlinkLoop = (codexName: string): boolean => {
const agentSkillDir = path.join(ROOT, '.agents', 'skills', codexName);
try {
return fs.realpathSync(agentSkillDir) === fs.realpathSync(ROOT);
} catch { return false; }
};
if (fs.existsSync(path.join(ROOT, 'SKILL.md.tmpl'))) {
if (!isSymlinkLoop('gstack')) {
skills.push({ dir: '.', codexName: 'gstack' });
}
skills.push({ dir: '.', codexName: 'gstack' });
}
for (const entry of fs.readdirSync(ROOT, { withFileTypes: true })) {
if (!entry.isDirectory() || entry.name.startsWith('.') || entry.name === 'node_modules') continue;
if (entry.name === 'codex') continue; // /codex is excluded from Codex output
if (!fs.existsSync(path.join(ROOT, entry.name, 'SKILL.md.tmpl'))) continue;
const codexName = entry.name.startsWith('gstack-') ? entry.name : `gstack-${entry.name}`;
if (isSymlinkLoop(codexName)) continue;
skills.push({ dir: entry.name, codexName });
}
return skills;
@@ -1967,7 +1984,9 @@ describe('Codex generation (--host codex)', () => {
});
test('--host codex --dry-run freshness', () => {
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--dry-run'], {
// Dry-run against the out-dir render: determinism/idempotency check
// (regenerating produces the same bytes the module-level render did).
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--dry-run', '--out-dir', EXTERNAL_OUT], {
cwd: ROOT,
stdout: 'pipe',
stderr: 'pipe',
@@ -1982,12 +2001,12 @@ describe('Codex generation (--host codex)', () => {
});
test('--host agents alias produces same output as --host codex', () => {
const codexResult = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--dry-run'], {
const codexResult = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--dry-run', '--out-dir', EXTERNAL_OUT], {
cwd: ROOT,
stdout: 'pipe',
stderr: 'pipe',
});
const agentsResult = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'agents', '--dry-run'], {
const agentsResult = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'agents', '--dry-run', '--out-dir', EXTERNAL_OUT], {
cwd: ROOT,
stdout: 'pipe',
stderr: 'pipe',
@@ -2195,63 +2214,52 @@ describe('Codex generation (--host codex)', () => {
// ─── Explicit --model override wins over the host default ────
// Without --model the codex host renders its defaultModel (gpt) — pinned by
// the golden test. This pins the OTHER direction through the real CLI:
// `./setup --host codex --model <id>` depends on it. Runs last in this
// describe and restores the host-default render before finishing.
// `./setup --host codex --model <id>` depends on it. The override renders
// into its OWN out-dir, so no restore pass is needed — the host-default
// render (EXTERNAL_OUT) is untouched and asserted directly.
test('explicit --model overrides the codex host default', () => {
const overrideOut = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-model-override-'));
try {
const override = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--model', 'claude'], {
const override = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--model', 'claude', '--out-dir', overrideOut], {
cwd: ROOT,
stdout: 'pipe',
stderr: 'pipe',
});
expect(override.exitCode).toBe(0);
const content = fs.readFileSync(path.join(AGENTS_DIR, 'gstack-ship', 'SKILL.md'), 'utf-8');
const content = fs.readFileSync(path.join(overrideOut, '.agents', 'skills', 'gstack-ship', 'SKILL.md'), 'utf-8');
expect(content).toContain('Model-Specific Behavioral Patch (claude)');
// The overlay now travels as --model into gstack-skill-start, which
// echoes MODEL_OVERLAY at runtime.
expect(content).toContain('--model "claude"');
} finally {
// Restore the host-default render — later tests and the host-config
// golden read this tree.
const restore = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'codex'], {
cwd: ROOT,
stdout: 'pipe',
stderr: 'pipe',
});
expect(restore.exitCode).toBe(0);
fs.rmSync(overrideOut, { recursive: true, force: true });
}
const restored = fs.readFileSync(path.join(AGENTS_DIR, 'gstack-ship', 'SKILL.md'), 'utf-8');
expect(restored).toContain('Model-Specific Behavioral Patch (gpt)');
expect(restored).toContain('--model "gpt"');
// Host-default direction: the untouched EXTERNAL_OUT render carries gpt.
const hostDefault = fs.readFileSync(path.join(AGENTS_DIR, 'gstack-ship', 'SKILL.md'), 'utf-8');
expect(hostDefault).toContain('Model-Specific Behavioral Patch (gpt)');
expect(hostDefault).toContain('--model "gpt"');
});
});
// ─── Factory generation tests ────────────────────────────────
describe('Factory generation (--host factory)', () => {
const FACTORY_DIR = path.join(ROOT, '.factory', 'skills');
// Generate Factory output for tests
Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'factory'], {
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
});
// .factory/ is gitignored — read the module-level out-dir render
// (--host all covers factory) instead of regenerating in place.
const FACTORY_DIR = path.join(EXTERNAL_OUT, '.factory', 'skills');
// Fresh out-dir → the vendored-dev-mode symlink loop can never occur, so
// every template renders (see the Codex discovery note above).
const FACTORY_SKILLS = (() => {
const skills: Array<{ dir: string; factoryName: string }> = [];
const isSymlinkLoop = (name: string): boolean => {
const factorySkillDir = path.join(ROOT, '.factory', 'skills', name);
try { return fs.realpathSync(factorySkillDir) === fs.realpathSync(ROOT); }
catch { return false; }
};
if (fs.existsSync(path.join(ROOT, 'SKILL.md.tmpl'))) {
if (!isSymlinkLoop('gstack')) skills.push({ dir: '.', factoryName: 'gstack' });
skills.push({ dir: '.', factoryName: 'gstack' });
}
for (const entry of fs.readdirSync(ROOT, { withFileTypes: true })) {
if (!entry.isDirectory() || entry.name.startsWith('.') || entry.name === 'node_modules') continue;
if (entry.name === 'codex') continue;
if (!fs.existsSync(path.join(ROOT, entry.name, 'SKILL.md.tmpl'))) continue;
const factoryName = entry.name.startsWith('gstack-') ? entry.name : `gstack-${entry.name}`;
if (isSymlinkLoop(factoryName)) continue;
skills.push({ dir: entry.name, factoryName });
}
return skills;
@@ -2333,10 +2341,10 @@ describe('Factory generation (--host factory)', () => {
});
test('--host droid alias works', () => {
const factoryResult = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'factory', '--dry-run'], {
const factoryResult = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'factory', '--dry-run', '--out-dir', EXTERNAL_OUT], {
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
});
const droidResult = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'droid', '--dry-run'], {
const droidResult = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'droid', '--dry-run', '--out-dir', EXTERNAL_OUT], {
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
});
expect(factoryResult.exitCode).toBe(0);
@@ -2345,7 +2353,7 @@ describe('Factory generation (--host factory)', () => {
});
test('--host factory --dry-run freshness', () => {
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'factory', '--dry-run'], {
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'factory', '--dry-run', '--out-dir', EXTERNAL_OUT], {
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
});
expect(result.exitCode).toBe(0);
@@ -2369,33 +2377,19 @@ describe('Factory generation (--host factory)', () => {
import { ALL_HOST_CONFIGS, getExternalHosts } from '../hosts/index';
describe('Parameterized host smoke tests', () => {
// Regenerate every external host up front so the per-host `--dry-run` freshness
// checks are deterministic. These host dirs (.agents/.factory/.cursor/...) are
// gitignored regenerated artifacts, so the freshness check is really an
// idempotency/determinism check — it still catches non-deterministic gen, but no
// longer flakes on stale-on-disk state left by a missing `gen --host all` prestep
// (the canonical `bun test` does not run one). The tracked-claude freshness test
// Every external host was rendered up front by the module-level
// `--host all --out-dir EXTERNAL_OUT` render, so the per-host `--dry-run`
// freshness checks are deterministic: they compare a regeneration against
// that render — an idempotency/determinism check that catches
// non-deterministic gen without ever writing (or depending on) the live
// gitignored host dirs. The tracked-claude freshness test
// (`generated files are fresh`) runs earlier and is unaffected.
beforeAll(() => {
for (const h of getExternalHosts()) {
Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', h.name], {
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
});
}
});
for (const hostConfig of getExternalHosts()) {
describe(`${hostConfig.displayName} (--host ${hostConfig.name})`, () => {
const hostDir = path.join(ROOT, hostConfig.hostSubdir, 'skills');
const hostDir = path.join(EXTERNAL_OUT, hostConfig.hostSubdir, 'skills');
test('generates output that exists on disk', () => {
// Generated dir should exist (created by earlier bun run gen:skill-docs --host all)
if (!fs.existsSync(hostDir)) {
// Generate if not already done
Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', hostConfig.name], {
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
});
}
// The module-level --host all render must have produced this host's tree.
expect(fs.existsSync(hostDir)).toBe(true);
const skills = fs.readdirSync(hostDir).filter(d =>
fs.existsSync(path.join(hostDir, d, 'SKILL.md'))
@@ -2437,7 +2431,7 @@ describe('Parameterized host smoke tests', () => {
test('--dry-run freshness check passes', () => {
const result = Bun.spawnSync(
['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', hostConfig.name, '--dry-run'],
['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', hostConfig.name, '--dry-run', '--out-dir', EXTERNAL_OUT],
{ cwd: ROOT, stdout: 'pipe', stderr: 'pipe' }
);
expect(result.exitCode).toBe(0);
@@ -2457,18 +2451,12 @@ describe('Parameterized host smoke tests', () => {
// ─── --host all tests ────────────────────────────────────────
describe('--host all', () => {
// Same determinism guard as the parameterized block: make external hosts fresh on
// disk so `--host all --dry-run` reports FRESH regardless of prior state.
beforeAll(() => {
for (const h of getExternalHosts()) {
Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', h.name], {
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
});
}
});
// Same determinism guard as the parameterized block: the module-level
// `--host all --out-dir EXTERNAL_OUT` render is the comparison baseline, so
// this dry-run reports FRESH regardless of live-tree state — and proves the
// claude host plus every external host regenerate deterministically.
test('--host all generates for all registered hosts', () => {
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'all', '--dry-run'], {
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'all', '--dry-run', '--out-dir', EXTERNAL_OUT], {
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
});
expect(result.exitCode).toBe(0);
@@ -2636,8 +2624,8 @@ describe('setup script validation', () => {
// T2: Dynamic $GSTACK_ROOT paths in generated Codex preambles
test('generated Codex preambles use dynamic GSTACK_ROOT paths', () => {
const codexSkillDir = path.join(ROOT, '.agents', 'skills', 'gstack-ship');
if (!fs.existsSync(codexSkillDir)) return; // skip if .agents/ not generated
// Read the module-level out-dir render (always present).
const codexSkillDir = path.join(EXTERNAL_OUT, '.agents', 'skills', 'gstack-ship');
const content = fs.readFileSync(path.join(codexSkillDir, 'SKILL.md'), 'utf-8');
expect(content).toContain('GSTACK_ROOT=');
expect(content).toContain('$GSTACK_BIN/');
@@ -3332,7 +3320,10 @@ describe('gen-skill-docs prefix warning (#620/#578)', () => {
fs.mkdirSync(fakeGstack, { recursive: true });
fs.writeFileSync(path.join(fakeGstack, 'config.yaml'), 'skill_prefix: true\n');
const output = execSync('bun run scripts/gen-skill-docs.ts', {
// Render into an out-dir under the fixture (the warning fires on any
// non-dry-run generation) so the live tree is never rewritten.
const outDir = path.join(tmpDir, 'out');
const output = execSync(`bun run scripts/gen-skill-docs.ts --out-dir "${outDir}"`, {
cwd: ROOT,
env: { ...process.env, HOME: fakeHome },
encoding: 'utf-8',
@@ -3353,7 +3344,8 @@ describe('gen-skill-docs prefix warning (#620/#578)', () => {
fs.mkdirSync(fakeGstack, { recursive: true });
fs.writeFileSync(path.join(fakeGstack, 'config.yaml'), 'skill_prefix: false\n');
const output = execSync('bun run scripts/gen-skill-docs.ts', {
const outDir = path.join(tmpDir, 'out');
const output = execSync(`bun run scripts/gen-skill-docs.ts --out-dir "${outDir}"`, {
cwd: ROOT,
env: { ...process.env, HOME: fakeHome },
encoding: 'utf-8',
@@ -3469,14 +3461,16 @@ describe('plan-mode-info resolver (handshake-replacement)', () => {
expect(checked).toBeGreaterThan(0);
});
test('vestigial handshake is absent from non-Claude host outputs when present on disk', () => {
test('vestigial handshake is absent from non-Claude host outputs', () => {
// Non-Claude hosts render to hostSubdirs (.agents/, .openclaw/, etc). The
// plan-mode-info resolver has no host-scoping — all hosts get the new
// section, none get the old handshake. Scan all candidate host dirs.
// section, none get the old handshake. Scan every candidate host tree in
// the module-level out-dir render (--host all), which is always present —
// so the check can no longer silently degrade to a console warning.
const hostDirs = ['.agents', '.openclaw', '.opencode', '.factory', '.hermes', '.kiro', '.cursor', '.slate'];
let checked = 0;
for (const host of hostDirs) {
const skillsRoot = path.join(ROOT, host, 'skills');
const skillsRoot = path.join(EXTERNAL_OUT, host, 'skills');
if (!fs.existsSync(skillsRoot)) continue;
const entries = fs.readdirSync(skillsRoot, { withFileTypes: true });
for (const entry of entries) {
@@ -3488,13 +3482,7 @@ describe('plan-mode-info resolver (handshake-replacement)', () => {
checked++;
}
}
if (checked === 0) {
// eslint-disable-next-line no-console
console.warn(
'plan-mode-info: no non-Claude host outputs found for cross-host absence check — ' +
'run `bun run gen:skill-docs --host all` to populate',
);
}
expect(checked).toBeGreaterThan(0);
});
test.each(REVIEW_SKILLS)(
+53
View File
@@ -0,0 +1,53 @@
/**
* No module-scope GSTACK_HOME assignment in any test file.
*
* Shard processes evaluate many test-file modules in one bun process, and a
* module can be loaded before its tests run — so a module-scope
* `process.env.GSTACK_HOME = ...` leaks into every sibling file in the
* shard. The damage was real before the 2026-08 sweep: relink.test.ts:28
* documents a "fresh install" test seeing a neighbor's skill_prefix, and
* cdp-e2e once baked a sibling's temp dir into artifacts that outlived it
* (dangling symlinks into a deleted render dir).
*
* The pattern is: save the original, assign in beforeAll, restore in
* afterAll — confining the value to the file's execution window. See
* browse/test/cdp-e2e.test.ts for the reference shape.
*
* Heuristic: repo test files write module-scope statements unindented, so a
* column-0 assignment is module scope; indented assignments (inside hooks,
* tests, or helpers) are fine.
*/
import { describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as path from 'node:path';
const ROOT = path.resolve(__dirname, '..');
function trackedTestFiles(): string[] {
const out = spawnSync('git', ['ls-files', '*.test.ts'], {
cwd: ROOT, encoding: 'utf-8',
});
if (out.status !== 0) throw new Error(`git ls-files failed: ${out.stderr}`);
return out.stdout.split('\n').filter(Boolean);
}
describe('GSTACK_HOME module-scope tripwire', () => {
test('no test file assigns process.env.GSTACK_HOME at module scope', () => {
const files = trackedTestFiles();
expect(files.length).toBeGreaterThan(100); // scan-rot guard
const offenders: string[] = [];
for (const rel of files) {
const lines = fs.readFileSync(path.join(ROOT, rel), 'utf-8').split('\n');
lines.forEach((line, i) => {
if (/^(?:process\.env\.GSTACK_HOME|process\.env\.GSTACK_STATE_ROOT)\s*=[^=]/.test(line)) {
offenders.push(`${rel}:${i + 1}${line.trim()}`);
}
});
}
expect(offenders,
`module-scope env assignment leaks across shard siblings — move into beforeAll + restore in afterAll:\n${offenders.join('\n')}`,
).toEqual([]);
});
});
+14 -2
View File
@@ -8,16 +8,28 @@
* timestamp + scope + reason + CI provenance.
*/
import { describe, test, expect, beforeEach } from 'bun:test';
import { describe, test, expect, beforeAll, beforeEach, afterAll } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { logBudgetOverride } from './budget-override';
const TMP_HOME = fs.mkdtempSync(path.join(os.tmpdir(), 'budget-override-test-'));
process.env.GSTACK_HOME = TMP_HOME;
const AUDIT_PATH = path.join(TMP_HOME, 'analytics', 'spend-overrides.jsonl');
// GSTACK_HOME is scoped to this file's execution window (beforeAll/afterAll),
// never set at module load: bun evaluates sibling modules before running
// their tests, so a module-scope assignment leaks into every other file in
// the shard process (pinned by test/gstack-home-module-scope.test.ts).
const ORIGINAL_GSTACK_HOME = process.env.GSTACK_HOME;
beforeAll(() => {
process.env.GSTACK_HOME = TMP_HOME;
});
afterAll(() => {
if (ORIGINAL_GSTACK_HOME === undefined) delete process.env.GSTACK_HOME;
else process.env.GSTACK_HOME = ORIGINAL_GSTACK_HOME;
});
describe('logBudgetOverride', () => {
beforeEach(() => {
// Start each test with a clean audit file
+49 -1
View File
@@ -61,7 +61,55 @@ export function computeDiffSelection(
return selection.selected;
}
export let selectedTests: string[] | null = computeDiffSelection(E2E_TOUCHFILES, 'E2E'); // null = run all
/**
* Parse the sharded paid runner's precomputed selection (EVALS_SELECTION_JSON,
* written by serializePaidDiffSelection in scripts/test-paid-shards.ts).
* Returns { selected: null } for run-all. THROWS on any parse/shape failure —
* resolveModuleSelection turns that into a fail-open local recompute.
*/
export function parseEvalsSelectionJson(raw: string): { selected: string[] | null; reason: string } {
const parsed: unknown = JSON.parse(raw);
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error('not an object');
const { selected, reason } = parsed as { selected?: unknown; reason?: unknown };
if (selected !== null
&& !(Array.isArray(selected) && selected.every((s) => typeof s === 'string'))) {
throw new Error('selected must be null or string[]');
}
return {
selected: selected as string[] | null,
reason: typeof reason === 'string' ? reason : 'parent selection',
};
}
/**
* Resolve the module-load E2E selection: prefer the parent shard runner's
* EVALS_SELECTION_JSON — skipping this module's own git walk and, when
* touchfiles-data.ts is in the diff, the per-child bun subprocess that
* evaluates the old data file (test-selection.ts map-diff path, one per
* shard). On ANY parse/shape failure, fall back to computing locally
* (fail-open preserved) with one stderr warning.
*/
export function resolveModuleSelection(
raw: string | undefined,
compute: () => string[] | null,
stderrWrite: (text: string) => void = (text) => process.stderr.write(text),
): string[] | null {
if (raw) {
try {
const { selected, reason } = parseEvalsSelectionJson(raw);
stderrWrite(`\nE2E selection (parent-propagated: ${reason}): ${selected === null ? 'all' : selected.length} tests\n`);
return selected;
} catch (err) {
stderrWrite(`WARNING: malformed EVALS_SELECTION_JSON (${err instanceof Error ? err.message : String(err)}) — falling back to local selection\n`);
}
}
return compute();
}
export let selectedTests: string[] | null = resolveModuleSelection(
evalsEnabled ? process.env.EVALS_SELECTION_JSON : undefined,
() => computeDiffSelection(E2E_TOUCHFILES, 'E2E'),
); // null = run all
// EVALS_TIER: filter tests by tier after diff-based selection.
// 'gate' = gate tests only (CI default — blocks merge)
+43
View File
@@ -0,0 +1,43 @@
/**
* Timeout policy for paid tests — five tiers instead of hand-tuned sprawl.
*
* Before this module the paid suite carried 46×300s, 46×120s, 44×360s,
* 44×180s, 27×240s, 19×150s, 13×420s, 12×600s, 7×700s… hand-ratcheted
* per test, several inflated to paper over the old 40-way in-shard
* concurrency (session startup queued behind 39 siblings and ate the
* budget before turn one — dead with the sharded runner's 1-file-per-shard
* model). Pick the tier that matches the test's SHAPE; escape-hatch raw
* literals stay legal with a justification comment (count-ratcheted by
* test/eval-budgets-policy.test.ts).
*
* Every tier must fit inside the lane walls — pinned by the fit test in
* test/eval-budgets-policy.test.ts against the sharded runner's
* DEFAULT_SHARD_TIMEOUT_MS. Budget above the wall is fiction, not headroom.
*/
/** LLM-judge call over an existing capture (no agent session). */
export const JUDGE_MS = 120_000;
/** One `claude -p` / SDK capture, bounded turns. */
export const CAPTURE_MS = 300_000;
/** Multi-capture or long multi-turn `claude -p` flows. */
export const CAPTURE_LONG_MS = 600_000;
/** Interactive real-PTY flow (spawn + skill + a few interactions). */
export const PTY_MS = 900_000;
/**
* Chained/judged PTY observation — the ceiling tier. 1200s leaves the
* 1800s shard wall real overhead; anything that genuinely needs more
* should be SPLIT, not budgeted past the wall.
*/
export const PTY_LONG_MS = 1_200_000;
export const ALL_TIERS = {
JUDGE_MS,
CAPTURE_MS,
CAPTURE_LONG_MS,
PTY_MS,
PTY_LONG_MS,
} as const;
+30 -12
View File
@@ -11,6 +11,8 @@
import Anthropic from '@anthropic-ai/sdk';
import { resolveEvalModel } from '../../lib/eval-model';
export interface JudgeScore {
clarity: number; // 1-5
completeness: number; // 1-5
@@ -52,9 +54,10 @@ export interface RecommendationScore {
/**
* Call an Anthropic model with a prompt, extract JSON response.
* Retries once on 429 rate limit errors. Defaults to Sonnet 4.6 for
* existing callers; pass a model id (e.g. claude-haiku-4-5-20251001)
* for cheaper bounded judgments like judgeRecommendation.
* Jittered exponential backoff over three 429 retries. Model resolves via
* lib/eval-model's `judge` kind (Sonnet default); pass a model id
* (e.g. claude-haiku-4-5-20251001) for cheaper bounded judgments like
* judgeRecommendation.
*/
// Default judge model: Sonnet. D1a tried Haiku 4.5 here and the first live
// run regressed the doc-rubric family — a controlled A/B on the identical
@@ -68,27 +71,42 @@ export interface RecommendationScore {
// distill — see lib/eval-model.ts).
export async function callJudge<T>(
prompt: string,
model: string = process.env.GSTACK_EVAL_MODEL_JUDGE || 'claude-sonnet-4-6',
model?: string,
opts?: { temperature?: number; max_tokens?: number },
): Promise<T> {
// Routed through the documented single resolution point: explicit arg >
// GSTACK_EVAL_MODEL_JUDGE > GSTACK_EVAL_MODEL > sonnet default. The old
// inline `GSTACK_EVAL_MODEL_JUDGE || sonnet` silently ignored the global
// GSTACK_EVAL_MODEL override that every other eval call site honors.
// opts (temperature/max_tokens) exist for bounded judgments like armJudge;
// defaults preserve prior behavior.
const resolvedModel = resolveEvalModel('judge', model);
const client = new Anthropic();
const makeRequest = () => client.messages.create({
model,
model: resolvedModel,
max_tokens: opts?.max_tokens ?? 1024,
...(opts?.temperature !== undefined ? { temperature: opts.temperature } : {}),
messages: [{ role: 'user', content: prompt }],
});
// 429s under CI concurrency: jittered exponential backoff over 3 retries
// (~1s/4s/16s + jitter), honoring the server's retry-after when present.
// The old single fixed 1s retry lost races reliably at 40-way concurrency.
let response;
try {
response = await makeRequest();
} catch (err: any) {
if (err.status === 429) {
await new Promise(r => setTimeout(r, 1000));
let attempt = 0;
for (;;) {
try {
response = await makeRequest();
} else {
throw err;
break;
} catch (err: any) {
if (err?.status !== 429 || attempt >= 3) throw err;
const retryAfterSecs = Number(err?.headers?.['retry-after']);
const baseMs = Number.isFinite(retryAfterSecs) && retryAfterSecs > 0
? retryAfterSecs * 1000
: 1000 * 4 ** attempt;
await new Promise((r) => setTimeout(r, baseMs + Math.random() * 500));
attempt += 1;
}
}
+11 -3
View File
@@ -11,12 +11,20 @@ import { matchGlob } from './touchfiles';
/** The exact globs package.json's `test:gate` passes to `bun test`. */
export const PAID_TEST_GLOBS = [
'test/skill-llm-eval.test.ts',
// skill-llm-eval* (not just the base file): skill-llm-eval-spec.test.ts
// fell outside the exact glob and could never run in any lane.
'test/skill-llm-eval*.test.ts',
'test/skill-e2e-*.test.ts',
'test/skill-routing-e2e.test.ts',
'test/codex-e2e.test.ts',
'test/codex-e2e-sol-scope.test.ts',
// codex-e2e* (was two exact names): codex-e2e-plan-format.test.ts and
// codex-e2e-recommendation-substance.test.ts were API-spending orphans —
// outside these globs they self-skipped in the free suite AND never
// entered the paid census. The same bug class as the deleted pre-split
// monolith (see test/paid-shards.test.ts's regression pin).
'test/codex-e2e*.test.ts',
'test/gemini-e2e.test.ts',
'test/llm-judge-recommendation.test.ts',
'test/carve-section-loading.test.ts',
] as const;
/** True when a repo-relative path (either slash style) is a paid test file. */
+34
View File
@@ -0,0 +1,34 @@
/**
* Periodic-lane exclusions — LITERALS ONLY (own file, deliberately NOT in
* touchfiles-data.ts: that file is evaluated standalone by map-diff against
* old git versions, and its contract must not grow unrelated exports).
*
* The weekly periodic CI lane runs EVERY periodic-tier file (EVALS_ALL=1) so
* tests can't rot invisibly — the coverage contract. A file lands here only
* when running it weekly is KNOWN waste (documented-red or requires manual
* hardware), and every entry must carry a tracking pointer with a re-entry
* condition, so an exclusion is a decision with an owner, not a place tests
* go to die. Pinned by test/periodic-exclude-policy.test.ts: entries must
* name real files and carry non-empty reason + tracking.
*
* Removing an entry re-activates the file on the next weekly run — that IS
* the re-entry mechanism.
*/
export const PERIODIC_CI_EXCLUDE: Record<string, { reason: string; tracking: string }> = {
'test/skill-e2e-ship-idempotency.test.ts': {
reason:
'documented-red: the PTY child sits at the Claude Code welcome screen for the full budget '
+ '(readiness/typing race vs CLI 2.1.x); never green since it was born in v1.63',
tracking: 'TODOS.md "periodic tier — three documented-red tests need structural repair" (1 of 3 resolved: sidebar trio already deleted)',
},
'test/skill-e2e-brain-privacy-gate.test.ts': {
reason:
'documented-red: the artifacts-sync stop-gate preconditions do not survive the hermetic env '
+ 'even with per-test HOME/GSTACK_HOME injection; never green anywhere',
tracking: 'TODOS.md "periodic tier — three documented-red tests need structural repair"',
},
'test/skill-e2e-ios.test.ts': {
reason: 'requires a live iOS device/simulator toolchain (xcodebuild, devicectl) — manual hardware, not a CI runner capability',
tracking: 'TODOS.md "skill-e2e-ios CI story" (device/runner decision)',
},
};
+39
View File
@@ -0,0 +1,39 @@
import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { runBin } from './run-bin';
describe('runBin', () => {
test('captures status/stdout/stderr with utf-8 shaping', () => {
const r = runBin('sh', ['-c', 'printf out; printf err >&2; exit 3']);
expect(r).toEqual({ status: 3, stdout: 'out', stderr: 'err' });
});
test('gstackHome sets both GSTACK_HOME and GSTACK_STATE_DIR (config precedence)', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'run-bin-'));
try {
const r = runBin('sh', ['-c', 'printf "%s|%s" "$GSTACK_HOME" "$GSTACK_STATE_DIR"'], { gstackHome: dir });
expect(r.stdout).toBe(`${dir}|${dir}`);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
test('env undefined deletes a key; input feeds stdin; trim shapes output', () => {
const r = runBin('sh', ['-c', 'cat; printf " padded "; test -z "$LANG" && printf noLANG >&2'], {
env: { LANG: undefined },
input: 'piped|',
trim: true,
});
// trim shapes the ENDS of the whole stream; interior whitespace stays.
expect(r.stdout).toBe('piped| padded');
expect(r.stderr).toBe('noLANG');
});
test('spawn failure yields -1, never a fake success', () => {
const r = runBin('/definitely/not/a/binary');
expect(r.status).toBe(-1);
});
});
+71
View File
@@ -0,0 +1,71 @@
/**
* Shared spawnSync wrapper for free unit tests that shell out to bin/
* scripts. Before this helper, ~36 test files each carried a near-identical
* local `run()` (spawnSync + utf-8 + {status, stdout, stderr} normalization)
* differing only in env composition, cwd, and timeout — drift-prone copies
* of one idea.
*
* Free-test-only by design: nothing under the paid globs should import this,
* so it never becomes a de facto global touchfile (paid selection is owned
* by test/helpers/e2e-helpers.ts and friends).
*/
import { spawnSync } from 'node:child_process';
export interface RunBinResult {
status: number;
stdout: string;
stderr: string;
}
export interface RunBinOptions {
cwd?: string;
/** Merged over process.env (an `undefined` value deletes the key). */
env?: Record<string, string | undefined>;
/**
* Isolation shorthand: sets GSTACK_HOME + GSTACK_STATE_DIR (gstack-config
* precedence is GSTACK_HOME > GSTACK_STATE_DIR > $HOME/.gstack, so both
* must move to isolate from the operator's real ~/.gstack).
*/
gstackHome?: string;
/** Also move $HOME (bins that write $HOME-anchored files, e.g. artifacts-remote pointers). */
home?: string;
input?: string;
/** Default 60s — a wedged bin fails the test, never the shard wall. */
timeoutMs?: number;
maxBuffer?: number;
/** Trim stdout/stderr (config-getter style bins). */
trim?: boolean;
}
export function runBin(command: string, args: string[] = [], opts: RunBinOptions = {}): RunBinResult {
const env: Record<string, string | undefined> = { ...process.env, ...opts.env };
if (opts.gstackHome !== undefined) {
env.GSTACK_HOME = opts.gstackHome;
env.GSTACK_STATE_DIR = opts.gstackHome;
}
if (opts.home !== undefined) env.HOME = opts.home;
for (const key of Object.keys(env)) {
if (env[key] === undefined) delete env[key];
}
const result = spawnSync(command, args, {
cwd: opts.cwd,
env: env as Record<string, string>,
encoding: 'utf-8',
input: opts.input,
timeout: opts.timeoutMs ?? 60_000,
maxBuffer: opts.maxBuffer,
});
const shape = (text: string | null | undefined): string => {
const value = text ?? '';
return opts.trim ? value.trim() : value;
};
return {
// -1 for spawn failure/kill mirrors the strictest of the old locals: a
// null status must never alias a real exit code.
status: result.status ?? -1,
stdout: shape(result.stdout),
stderr: shape(result.stderr),
};
}
+135 -130
View File
@@ -22,8 +22,8 @@
*/
export const E2E_TOUCHFILES: Record<string, string[]> = {
// Browse core (+ test-server dependency)
'browse-basic': ['browse/src/**', 'browse/test/test-server.ts'],
'browse-snapshot': ['browse/src/**', 'browse/test/test-server.ts'],
'browse-basic': ['browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-bws.test.ts'],
'browse-snapshot': ['browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-bws.test.ts'],
// Hermetic isolation canaries (hermetic-env.ts is also a GLOBAL touchfile;
// these entries exist so the canaries themselves stay tier-classified)
@@ -37,21 +37,21 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'first-task-scaffold': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-first-task-detect', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'test/skill-e2e-first-task-scaffold.test.ts', 'test/helpers/session-runner.ts'],
// SKILL.md setup + preamble (depend on ROOT SKILL.md + gen-skill-docs)
'skillmd-setup-discovery': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'skillmd-no-local-binary': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'skillmd-outside-git': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'skillmd-setup-discovery': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'],
'skillmd-no-local-binary': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'],
'skillmd-outside-git': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'],
'session-awareness': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'operational-learning': ['scripts/resolvers/preamble.ts', 'bin/gstack-learnings-log'],
'session-awareness': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'],
'operational-learning': ['scripts/resolvers/preamble.ts', 'bin/gstack-learnings-log', 'test/skill-e2e-bws.test.ts'],
// QA (+ test-server dependency)
'qa-quick': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts'],
'qa-b6-static': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval.html', 'test/fixtures/qa-eval-ground-truth.json'],
'qa-b7-spa': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-spa.html', 'test/fixtures/qa-eval-spa-ground-truth.json'],
'qa-b8-checkout': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-checkout.html', 'test/fixtures/qa-eval-checkout-ground-truth.json'],
'qa-only-no-fix': ['qa-only/**', 'qa/templates/**'],
'qa-fix-loop': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts'],
'qa-bootstrap': ['qa/**', 'ship/**'],
'qa-quick': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts'],
'qa-b6-static': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval.html', 'test/fixtures/qa-eval-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts'],
'qa-b7-spa': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-spa.html', 'test/fixtures/qa-eval-spa-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts'],
'qa-b8-checkout': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-checkout.html', 'test/fixtures/qa-eval-checkout-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts'],
'qa-only-no-fix': ['qa-only/**', 'qa/templates/**', 'test/skill-e2e-qa-workflow.test.ts'],
'qa-fix-loop': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts'],
'qa-bootstrap': ['qa/**', 'ship/**', 'test/skill-e2e-qa-workflow.test.ts'],
// Review
'review-sql-injection': ['review/**', 'test/fixtures/review-eval-vuln.rb', 'test/skill-e2e-review.test.ts'],
@@ -60,29 +60,29 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'review-design-lite': ['review/**', 'test/fixtures/review-eval-design-slop.*', 'test/skill-e2e-review.test.ts'],
// Review Army (specialist dispatch)
'review-army-migration-safety': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope'],
'review-army-perf-n-plus-one': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope'],
'review-army-delivery-audit': ['review/**', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts'],
'review-army-quality-score': ['review/**', 'scripts/resolvers/review-army.ts'],
'review-army-json-findings': ['review/**', 'scripts/resolvers/review-army.ts'],
'review-army-red-team': ['review/**', 'scripts/resolvers/review-army.ts'],
'review-army-migration-safety': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope', 'test/skill-e2e-review-army.test.ts'],
'review-army-perf-n-plus-one': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope', 'test/skill-e2e-review-army.test.ts'],
'review-army-delivery-audit': ['review/**', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts'],
'review-army-quality-score': ['review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts'],
'review-army-json-findings': ['review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts'],
'review-army-red-team': ['review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts'],
'review-army-simplification': ['review/**', 'scripts/resolvers/review-army.ts', 'test/fixtures/review-army-overbuild.js', 'test/fixtures/review-army-lean-complete.js', 'test/skill-e2e-review-army.test.ts'],
'review-army-simplification-precision': ['review/**', 'scripts/resolvers/review-army.ts', 'test/fixtures/review-army-overbuild.js', 'test/fixtures/review-army-lean-complete.js', 'test/skill-e2e-review-army.test.ts'],
'review-army-consensus': ['review/**', 'scripts/resolvers/review-army.ts'],
'review-army-consensus': ['review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts'],
// Office Hours
'office-hours-spec-review': ['office-hours/**', 'scripts/gen-skill-docs.ts'],
'office-hours-forcing-energy': ['office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts'],
'office-hours-builder-wildness': ['office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts'],
'office-hours-spec-review': ['office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'],
'office-hours-forcing-energy': ['office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours.test.ts'],
'office-hours-builder-wildness': ['office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours.test.ts'],
// Plan reviews
'plan-ceo-review': ['plan-ceo-review/**'],
'plan-ceo-review-selective': ['plan-ceo-review/**'],
'plan-ceo-review-benefits': ['plan-ceo-review/**', 'scripts/gen-skill-docs.ts'],
'plan-ceo-review-expansion-energy': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts'],
'plan-eng-review': ['plan-eng-review/**'],
'plan-eng-review-artifact': ['plan-eng-review/**'],
'plan-review-report': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts'],
'plan-ceo-review': ['plan-ceo-review/**', 'test/skill-e2e-plan.test.ts'],
'plan-ceo-review-selective': ['plan-ceo-review/**', 'test/skill-e2e-plan.test.ts'],
'plan-ceo-review-benefits': ['plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'],
'plan-ceo-review-expansion-energy': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan.test.ts'],
'plan-eng-review': ['plan-eng-review/**', 'test/skill-e2e-plan.test.ts'],
'plan-eng-review-artifact': ['plan-eng-review/**', 'test/skill-e2e-plan.test.ts'],
'plan-review-report': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'],
// Plan-mode smoke tests — gate-tier safety regression tests. Each test file
// contains TWO test cases as of v1.21: the baseline plan-mode case and the
@@ -93,7 +93,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
// regression test outcome between 'asked' and 'auto_decided'.
'plan-ceo-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'],
'plan-eng-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'],
'plan-design-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts'],
'plan-design-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts', 'test/skill-e2e-design.test.ts'],
'plan-devex-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts'],
// Covers ceo (preamble misfire) + eng/design (scope-gate bypass must not
// fire outside plan mode) + the named-target exception case. 4 PTY runs;
@@ -119,7 +119,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
// written a never-ask preference, AUQ should still auto-decide rather than
// surfacing the question. Touches the question-tuning + preference
// infrastructure plus the resolvers that own the AUTO_DECIDE preamble.
'auto-decide-preserved': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts'],
'auto-decide-preserved': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-auto-decide-preserved.test.ts'],
// Conductor → prose decision brief (Conductor signal makes prose the default;
// the PreToolUse hook denies the flaky tool). Touches the resolver that owns
@@ -130,7 +130,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
// Each one tests behavior the SDK harness can't observe (rendered TTY,
// numbered-option lists, multi-phase ordering, idempotency state echo).
'preamble-script-ab': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'plan-ceo-review/**', 'test/helpers/auq-sdk-capture.ts', 'test/skill-e2e-preamble-script-ab.test.ts'],
'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts'],
'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-ask-user-question-format-compliance.test.ts'],
'auq-repetition-cut-ab': ['scripts/resolvers/preamble/generate-ask-user-format.ts', 'plan-ceo-review/**', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/llm-judge.ts', 'test/fixtures/auq-pre-cut-plan-ceo-review-SKILL.md', 'test/skill-e2e-auq-repetition-cut-ab.test.ts'],
'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'],
'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'],
@@ -142,12 +142,12 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'tpa-absent-darwin': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
'tpa-apple-ban': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
'ship-section-loading': ['ship/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-ship-section-loading.test.ts'],
'plan-ceo-section-loading': ['plan-ceo-review/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
'plan-ceo-section-loading': ['plan-ceo-review/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-plan-ceo-review-section-loading.test.ts'],
// Data-driven behavioral guard for the 'plan'/'prompt' carves (eng, design,
// devex, office-hours + future PR2 carves). One file iterating CARVE_GUARDS;
// the selector sets GSTACK_CARVE_SKILL=<name> to scope cost to the changed
// skill (D-CODEX A). Touching the registry/helper or sections.ts runs all.
'carve-section-loading': ['design-html/**', 'design-shotgun/**', 'qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
'carve-section-loading': ['design-html/**', 'design-shotgun/**', 'qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/carve-section-loading.test.ts'],
'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-autoplan-chain.test.ts'],
'e2e-harness-audit': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/claude-pty-runner.ts'],
@@ -193,18 +193,18 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
// AskUserQuestion format regression (RECOMMENDATION + Completeness: N/10)
// Fires when either template OR the two preamble resolvers change.
'plan-ceo-review-format-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'],
'plan-ceo-review-format-approach': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'],
'plan-eng-review-format-coverage': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'],
'plan-eng-review-format-kind': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'],
'plan-ceo-review-format-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts'],
'plan-ceo-review-format-approach': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts'],
'plan-eng-review-format-coverage': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts'],
'plan-eng-review-format-kind': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts'],
// v1.7.0.0 Pros/Cons format cadence + format + negative-escape evals.
// Dependencies: same as format-mode + the 4 plan-review templates + overlay.
// All periodic-tier (non-deterministic Opus 4.7 behavior).
'plan-ceo-review-prosons-cadence': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
'plan-review-prosons-format': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
'plan-review-prosons-hardstop-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
'plan-review-prosons-neutral-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
'plan-ceo-review-prosons-cadence': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/skill-e2e-plan-prosons.test.ts'],
'plan-review-prosons-format': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/skill-e2e-plan-prosons.test.ts'],
'plan-review-prosons-hardstop-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/skill-e2e-plan-prosons.test.ts'],
'plan-review-prosons-neutral-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/skill-e2e-plan-prosons.test.ts'],
// Expanded coverage (CT3) — 6 non-plan-review skills inherit Pros/Cons via preamble
'ship-prosons-format': ['ship/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
@@ -216,24 +216,24 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'document-release-prosons-format': ['document-release/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
// /plan-tune (v1 observational)
'plan-tune-inspect': ['plan-tune/**', 'scripts/question-registry.ts', 'scripts/psychographic-signals.ts', 'scripts/one-way-doors.ts', 'bin/gstack-question-log', 'bin/gstack-question-preference', 'bin/gstack-developer-profile'],
'plan-tune-inspect': ['plan-tune/**', 'scripts/question-registry.ts', 'scripts/psychographic-signals.ts', 'scripts/one-way-doors.ts', 'bin/gstack-question-log', 'bin/gstack-question-preference', 'bin/gstack-developer-profile', 'test/skill-e2e-plan-tune.test.ts'],
// /plan-tune cathedral (T16 — 5 E2E scenarios, all gate per D12)
'plan-tune-hook-capture': ['hosts/claude/hooks/**', 'bin/gstack-question-log', 'bin/gstack-developer-profile', 'plan-tune/**'],
'plan-tune-enforcement': ['hosts/claude/hooks/**', 'bin/gstack-question-preference', 'scripts/question-registry.ts'],
'plan-tune-annotation': ['hosts/claude/hooks/**', 'scripts/declared-annotation.ts', 'scripts/psychographic-signals.ts', 'scripts/question-registry.ts'],
'plan-tune-codex-import': ['bin/gstack-codex-session-import', 'bin/gstack-question-log', 'docs/spikes/codex-session-format.md'],
'plan-tune-dream-cycle': ['bin/gstack-distill-free-text', 'bin/gstack-distill-apply', 'hosts/claude/hooks/**', 'plan-tune/**'],
'plan-tune-hook-capture': ['hosts/claude/hooks/**', 'bin/gstack-question-log', 'bin/gstack-developer-profile', 'plan-tune/**', 'test/skill-e2e-plan-tune-cathedral.test.ts'],
'plan-tune-enforcement': ['hosts/claude/hooks/**', 'bin/gstack-question-preference', 'scripts/question-registry.ts', 'test/skill-e2e-plan-tune-cathedral.test.ts'],
'plan-tune-annotation': ['hosts/claude/hooks/**', 'scripts/declared-annotation.ts', 'scripts/psychographic-signals.ts', 'scripts/question-registry.ts', 'test/skill-e2e-plan-tune-cathedral.test.ts'],
'plan-tune-codex-import': ['bin/gstack-codex-session-import', 'bin/gstack-question-log', 'docs/spikes/codex-session-format.md', 'test/skill-e2e-plan-tune-cathedral.test.ts'],
'plan-tune-dream-cycle': ['bin/gstack-distill-free-text', 'bin/gstack-distill-apply', 'hosts/claude/hooks/**', 'plan-tune/**', 'test/skill-e2e-plan-tune-cathedral.test.ts'],
// Codex offering verification
'codex-offered-office-hours': ['office-hours/**', 'scripts/gen-skill-docs.ts'],
'codex-offered-ceo-review': ['plan-ceo-review/**', 'scripts/gen-skill-docs.ts'],
'codex-offered-design-review': ['plan-design-review/**', 'scripts/gen-skill-docs.ts'],
'codex-offered-eng-review': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts'],
'codex-offered-office-hours': ['office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'],
'codex-offered-ceo-review': ['plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'],
'codex-offered-design-review': ['plan-design-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'],
'codex-offered-eng-review': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'],
// Ship
'ship-base-branch': ['ship/**', 'bin/gstack-repo-mode', 'test/skill-e2e-review-attribution.test.ts'],
'ship-local-workflow': ['ship/**', 'scripts/gen-skill-docs.ts'],
'ship-local-workflow': ['ship/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-workflow.test.ts'],
'review-dashboard-via': ['ship/**', 'scripts/resolvers/review.ts', 'codex/**', 'autoplan/**', 'land-and-deploy/**', 'test/skill-e2e-review-attribution.test.ts'],
// Retro
@@ -244,51 +244,51 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'global-discover': ['bin/gstack-global-discover.ts', 'test/global-discover.test.ts'],
// CSO
'cso-full-audit': ['cso/**'],
'cso-diff-mode': ['cso/**'],
'cso-infra-scope': ['cso/**'],
'cso-full-audit': ['cso/**', 'test/skill-e2e-cso.test.ts'],
'cso-diff-mode': ['cso/**', 'test/skill-e2e-cso.test.ts'],
'cso-infra-scope': ['cso/**', 'test/skill-e2e-cso.test.ts'],
// Learnings
'learnings-show': ['learn/**', 'bin/gstack-learnings-search', 'bin/gstack-learnings-log', 'scripts/resolvers/learnings.ts'],
'learnings-show': ['learn/**', 'bin/gstack-learnings-search', 'bin/gstack-learnings-log', 'scripts/resolvers/learnings.ts', 'test/skill-e2e-learnings.test.ts'],
// Session Intelligence (timeline, context recovery, /context-save + /context-restore)
'timeline-event-flow': ['bin/gstack-timeline-log', 'bin/gstack-timeline-read'],
'context-recovery-artifacts': ['scripts/resolvers/preamble.ts', 'bin/gstack-timeline-log', 'bin/gstack-slug', 'learn/**'],
'context-save-writes-file': ['context-save/**', 'bin/gstack-slug'],
'context-restore-loads-latest': ['context-restore/**', 'bin/gstack-slug'],
'timeline-event-flow': ['bin/gstack-timeline-log', 'bin/gstack-timeline-read', 'test/skill-e2e-session-intelligence.test.ts'],
'context-recovery-artifacts': ['scripts/resolvers/preamble.ts', 'bin/gstack-timeline-log', 'bin/gstack-slug', 'learn/**', 'test/skill-e2e-session-intelligence.test.ts'],
'context-save-writes-file': ['context-save/**', 'bin/gstack-slug', 'test/skill-e2e-session-intelligence.test.ts'],
'context-restore-loads-latest': ['context-restore/**', 'bin/gstack-slug', 'test/skill-e2e-session-intelligence.test.ts'],
// Context skills E2E (live-fire, Skill-tool routing path) — see
// test/skill-e2e-context-skills.test.ts. These are periodic-tier because
// each one spawns claude -p and costs ~$0.20-$0.40. Collectively they
// verify the thing the /checkpoint → /context-save rename was for.
'context-save-routing': ['context-save/**', 'scripts/resolvers/preamble.ts'],
'context-save-then-restore-roundtrip': ['context-save/**', 'context-restore/**', 'bin/gstack-slug'],
'context-restore-fragment-match': ['context-restore/**'],
'context-restore-empty-state': ['context-restore/**'],
'context-restore-list-delegates': ['context-restore/**'],
'context-restore-legacy-compat': ['context-restore/**'],
'context-save-list-current-branch': ['context-save/**'],
'context-save-list-all-branches': ['context-save/**'],
'context-save-routing': ['context-save/**', 'scripts/resolvers/preamble.ts', 'test/skill-e2e-context-skills.test.ts'],
'context-save-then-restore-roundtrip': ['context-save/**', 'context-restore/**', 'bin/gstack-slug', 'test/skill-e2e-context-skills.test.ts'],
'context-restore-fragment-match': ['context-restore/**', 'test/skill-e2e-context-skills.test.ts'],
'context-restore-empty-state': ['context-restore/**', 'test/skill-e2e-context-skills.test.ts'],
'context-restore-list-delegates': ['context-restore/**', 'test/skill-e2e-context-skills.test.ts'],
'context-restore-legacy-compat': ['context-restore/**', 'test/skill-e2e-context-skills.test.ts'],
'context-save-list-current-branch': ['context-save/**', 'test/skill-e2e-context-skills.test.ts'],
'context-save-list-all-branches': ['context-save/**', 'test/skill-e2e-context-skills.test.ts'],
// Document-release
'document-release': ['document-release/**'],
'document-release': ['document-release/**', 'test/skill-e2e-workflow.test.ts'],
// Codex (Claude E2E — tests /codex skill via Claude)
'codex-review': ['codex/**'],
'codex-review': ['codex/**', 'test/skill-e2e-workflow.test.ts'],
// Codex E2E (tests skills via Codex CLI + worktree)
'codex-discover-skill': ['codex/**', '.agents/skills/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts'],
'codex-review-findings': ['review/**', '.agents/skills/gstack-review/**', 'codex/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts'],
'codex-discover-skill': ['codex/**', '.agents/skills/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts', 'test/codex-e2e.test.ts'],
'codex-review-findings': ['review/**', '.agents/skills/gstack-review/**', 'codex/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts', 'test/codex-e2e.test.ts'],
// GPT-5.6 Sol scope-termination E2E (Codex CLI, full generated investigate skill)
'codex-sol-scope-termination': ['model-overlays/gpt-5.6-sol.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts', 'scripts/resolvers/preamble/**', 'investigate/**', 'test/helpers/codex-session-runner.ts', 'test/codex-e2e-sol-scope.test.ts'],
// Gemini E2E — smoke test only (Gemini gets lost in worktrees on complex tasks)
'gemini-smoke': ['.agents/skills/**', 'test/helpers/gemini-session-runner.ts', 'lib/worktree.ts'],
'gemini-smoke': ['.agents/skills/**', 'test/helpers/gemini-session-runner.ts', 'lib/worktree.ts', 'test/gemini-e2e.test.ts'],
// Coverage audit (shared fixture) + triage + gates
'ship-coverage-audit': ['ship/**', 'test/fixtures/coverage-audit-fixture.ts', 'bin/gstack-repo-mode'],
'ship-coverage-audit': ['ship/**', 'test/fixtures/coverage-audit-fixture.ts', 'bin/gstack-repo-mode', 'test/skill-e2e-workflow.test.ts'],
'review-coverage-audit': ['review/**', 'test/fixtures/coverage-audit-fixture.ts', 'test/skill-e2e-coverage-audit.test.ts'],
'plan-eng-coverage-audit': ['plan-eng-review/**', 'test/fixtures/coverage-audit-fixture.ts', 'test/skill-e2e-coverage-audit.test.ts'],
'ship-triage': ['ship/**', 'bin/gstack-repo-mode', 'test/skill-e2e-triage.test.ts'],
@@ -300,12 +300,12 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'review-plan-completion': ['review/**', 'scripts/gen-skill-docs.ts'],
// Design
'design-consultation-core': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/helpers/llm-judge.ts'],
'design-consultation-existing': ['design-consultation/**', 'scripts/gen-skill-docs.ts'],
'design-consultation-research': ['design-consultation/**', 'scripts/gen-skill-docs.ts'],
'design-consultation-preview': ['design-consultation/**', 'scripts/gen-skill-docs.ts'],
'plan-design-review-no-ui-scope': ['plan-design-review/**', 'scripts/gen-skill-docs.ts'],
'design-review-fix': ['design-review/**', 'browse/src/**', 'scripts/gen-skill-docs.ts'],
'design-consultation-core': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-design.test.ts'],
'design-consultation-existing': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
'design-consultation-research': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
'design-consultation-preview': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
'plan-design-review-no-ui-scope': ['plan-design-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
'design-review-fix': ['design-review/**', 'browse/src/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
// Design Shotgun
'design-shotgun-path': ['design-shotgun/**', 'design/src/**', 'scripts/resolvers/design.ts'],
@@ -314,24 +314,24 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
// /diagram (diagram-render bundle consumers). Triplet = deterministic
// functional (gate); authoring quality = LLM-judged benchmark (periodic).
'diagram-triplet': ['diagram/**', 'lib/diagram-render/**', 'browse/src/write-commands.ts', 'browse/src/read-commands.ts'],
'diagram-authoring-quality': ['diagram/**', 'lib/diagram-render/**', 'test/helpers/llm-judge.ts'],
'diagram-triplet': ['diagram/**', 'lib/diagram-render/**', 'browse/src/write-commands.ts', 'browse/src/read-commands.ts', 'test/skill-e2e-diagram.test.ts'],
'diagram-authoring-quality': ['diagram/**', 'lib/diagram-render/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-diagram.test.ts'],
// gstack-upgrade
'gstack-upgrade-happy-path': ['gstack-upgrade/**'],
'gstack-upgrade-happy-path': ['gstack-upgrade/**', 'test/skill-e2e-workflow.test.ts'],
// Deploy skills
'land-and-deploy-workflow': ['land-and-deploy/**', 'scripts/gen-skill-docs.ts'],
'land-and-deploy-first-run': ['land-and-deploy/**', 'scripts/gen-skill-docs.ts', 'bin/gstack-slug'],
'land-and-deploy-review-gate': ['land-and-deploy/**', 'bin/gstack-review-read'],
'canary-workflow': ['canary/**', 'browse/src/**'],
'benchmark-workflow': ['benchmark/**', 'browse/src/**'],
'setup-deploy-workflow': ['setup-deploy/**', 'scripts/gen-skill-docs.ts'],
'land-and-deploy-workflow': ['land-and-deploy/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-deploy.test.ts'],
'land-and-deploy-first-run': ['land-and-deploy/**', 'scripts/gen-skill-docs.ts', 'bin/gstack-slug', 'test/skill-e2e-deploy.test.ts'],
'land-and-deploy-review-gate': ['land-and-deploy/**', 'bin/gstack-review-read', 'test/skill-e2e-deploy.test.ts'],
'canary-workflow': ['canary/**', 'browse/src/**', 'test/skill-e2e-deploy.test.ts'],
'benchmark-workflow': ['benchmark/**', 'browse/src/**', 'test/skill-e2e-deploy.test.ts'],
'setup-deploy-workflow': ['setup-deploy/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-deploy.test.ts'],
// Autoplan
'autoplan-core': ['autoplan/**', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**'],
'autoplan-dual-voice': ['autoplan/**', 'codex/**', 'bin/gstack-codex-probe', 'scripts/resolvers/review.ts', 'scripts/resolvers/design.ts'],
'autoplan-dual-voice': ['autoplan/**', 'codex/**', 'bin/gstack-codex-probe', 'scripts/resolvers/review.ts', 'scripts/resolvers/design.ts', 'test/skill-e2e-autoplan-dual-voice.test.ts'],
// Multi-provider benchmark adapters — live API smoke against real claude/codex/gemini CLIs
'benchmark-providers-live': ['bin/gstack-model-benchmark', 'test/helpers/providers/**', 'test/helpers/benchmark-runner.ts', 'test/helpers/pricing.ts', 'test/skill-e2e-benchmark-providers.test.ts'],
@@ -344,41 +344,46 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'scrape-match-path': [
'scrape/**', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts',
'browser-skills/hackernews-frontpage/**',
'test/skill-e2e-skillify.test.ts',
],
'scrape-prototype-path': [
'scrape/**', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts',
'test/skill-e2e-skillify.test.ts',
],
'skillify-happy-path': [
'skillify/**', 'scrape/**', 'browse/src/browser-skill-write.ts',
'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts',
'test/skill-e2e-skillify.test.ts',
],
'skillify-provenance-refusal': [
'skillify/**', 'browse/src/browser-skill-write.ts',
'test/skill-e2e-skillify.test.ts',
],
'skillify-approval-reject': [
'skillify/**', 'scrape/**', 'browse/src/browser-skill-write.ts',
'test/skill-e2e-skillify.test.ts',
],
// Skill routing — journey-stage tests (depend on ALL skill descriptions)
'journey-ideation': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-plan-eng': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-debug': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-code-review': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-ship': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-docs': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-retro': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-design-system': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-visual-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'journey-ideation': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
'journey-plan-eng': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
'journey-debug': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
'journey-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
'journey-code-review': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
'journey-ship': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
'journey-docs': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
'journey-retro': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
'journey-design-system': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
'journey-visual-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
// Opus 4.7 behavior evals — keys match testName: values in the test file.
// Routing sub-tests use template literal `routing-${c.name}` testNames,
// which the touchfile completeness scanner skips; they inherit selection
// from the file-level touchfile entry via GLOBAL_TOUCHFILES.
'fanout-arm-overlay-on':
['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts'],
['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-opus-47.test.ts'],
'fanout-arm-overlay-off':
['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts'],
['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-opus-47.test.ts'],
// Overlay efficacy harness (SDK) — measures whether overlay nudges change
// behavior under @anthropic-ai/claude-agent-sdk (closer to real Claude Code
@@ -809,49 +814,49 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
* LLM-judge test touchfiles — keyed by test description string.
*/
export const LLM_JUDGE_TOUCHFILES: Record<string, string[]> = {
'command reference table': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts'],
'snapshot flags reference': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts'],
'browse/SKILL.md reference': ['browse/sections/**', 'browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**'],
'setup block': ['SKILL.md', 'SKILL.md.tmpl'],
'regression vs baseline': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json'],
'qa/SKILL.md workflow': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl'],
'qa/SKILL.md health rubric': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl'],
'qa/SKILL.md anti-refusal': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'],
'cross-skill greptile consistency': ['review/SKILL.md', 'review/SKILL.md.tmpl', 'ship/SKILL.md', 'ship/SKILL.md.tmpl', 'review/greptile-triage.md', 'retro/SKILL.md', 'retro/SKILL.md.tmpl'],
'baseline score pinning': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json'],
'command reference table': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/skill-llm-eval.test.ts'],
'snapshot flags reference': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts', 'test/skill-llm-eval.test.ts'],
'browse/SKILL.md reference': ['browse/sections/**', 'browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**', 'test/skill-llm-eval.test.ts'],
'setup block': ['SKILL.md', 'SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
'regression vs baseline': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json', 'test/skill-llm-eval.test.ts'],
'qa/SKILL.md workflow': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
'qa/SKILL.md health rubric': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
'qa/SKILL.md anti-refusal': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
'cross-skill greptile consistency': ['review/SKILL.md', 'review/SKILL.md.tmpl', 'ship/SKILL.md', 'ship/SKILL.md.tmpl', 'review/greptile-triage.md', 'retro/SKILL.md', 'retro/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
'baseline score pinning': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json', 'test/skill-llm-eval.test.ts'],
// Ship & Release
'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl'],
'document-release/SKILL.md workflow': ['document-release/SKILL.md', 'document-release/SKILL.md.tmpl'],
'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
'document-release/SKILL.md workflow': ['document-release/SKILL.md', 'document-release/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
// Plan Reviews
'plan-ceo-review/SKILL.md modes': ['plan-ceo-review/SKILL.md', 'plan-ceo-review/SKILL.md.tmpl'],
'plan-eng-review/SKILL.md sections': ['plan-eng-review/SKILL.md', 'plan-eng-review/SKILL.md.tmpl'],
'plan-ceo-review/SKILL.md modes': ['plan-ceo-review/SKILL.md', 'plan-ceo-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
'plan-eng-review/SKILL.md sections': ['plan-eng-review/SKILL.md', 'plan-eng-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
// /spec authored-spec quality (paid LLM-judge — periodic-tier).
'plan-design-review/SKILL.md passes': ['plan-design-review/SKILL.md', 'plan-design-review/SKILL.md.tmpl'],
'plan-design-review/SKILL.md passes': ['plan-design-review/SKILL.md', 'plan-design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
// Design skills
'design-review/SKILL.md fix loop': ['design-review/SKILL.md', 'design-review/SKILL.md.tmpl'],
'design-consultation/SKILL.md research': ['design-consultation/SKILL.md', 'design-consultation/SKILL.md.tmpl'],
'design-review/SKILL.md fix loop': ['design-review/SKILL.md', 'design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
'design-consultation/SKILL.md research': ['design-consultation/SKILL.md', 'design-consultation/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
// Office Hours
'office-hours/SKILL.md spec review': ['office-hours/SKILL.md', 'office-hours/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'office-hours/SKILL.md design sketch': ['office-hours/SKILL.md', 'office-hours/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
// Deploy skills
'land-and-deploy/SKILL.md workflow': ['land-and-deploy/SKILL.md', 'land-and-deploy/SKILL.md.tmpl', 'land-and-deploy/sections/**'],
'canary/SKILL.md monitoring loop': ['canary/SKILL.md', 'canary/SKILL.md.tmpl'],
'benchmark/SKILL.md perf collection': ['benchmark/SKILL.md', 'benchmark/SKILL.md.tmpl'],
'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl'],
'land-and-deploy/SKILL.md workflow': ['land-and-deploy/SKILL.md', 'land-and-deploy/SKILL.md.tmpl', 'land-and-deploy/sections/**', 'test/skill-llm-eval.test.ts'],
'canary/SKILL.md monitoring loop': ['canary/SKILL.md', 'canary/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
'benchmark/SKILL.md perf collection': ['benchmark/SKILL.md', 'benchmark/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
// Other skills
'retro/SKILL.md instructions': ['retro/sections/**', 'retro/SKILL.md', 'retro/SKILL.md.tmpl'],
'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'],
'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl'],
'retro/SKILL.md instructions': ['retro/sections/**', 'retro/SKILL.md', 'retro/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
// Voice directive
'voice directive tone': ['scripts/resolvers/preamble.ts', 'review/SKILL.md', 'review/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'voice directive tone': ['scripts/resolvers/preamble.ts', 'review/SKILL.md', 'review/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-llm-eval.test.ts'],
};
/**
+31 -21
View File
@@ -3,8 +3,9 @@
* host-config-export.ts, and golden-file regression checks.
*/
import { describe, test, expect, beforeAll } from 'bun:test';
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { validateHostConfig, validateAllConfigs, type HostConfig } from '../scripts/host-config';
import {
@@ -428,34 +429,43 @@ describe('host-config-export.ts CLI', () => {
describe('golden-file regression', () => {
const GOLDEN_DIR = path.join(ROOT, 'test', 'fixtures', 'golden');
// #2532: the codex/factory goldens read gitignored .agents/ and .factory/
// artifacts that only gen-skill-docs.test.ts (a serial tree-mutating file)
// produces. On a clean clone — or when this file runs in isolation — those
// dirs don't exist and the goldens fail with ENOENT, an order dependency,
// not a regression. Self-provision: generate a host's artifacts iff its
// ship SKILL.md is missing. Existing artifacts are never overwritten here,
// so a genuinely stale artifact still fails the golden (that is the test's
// job; freshness enforcement lives in gen-skill-docs.test.ts).
// #2532 successor: the codex/factory goldens used to read gitignored
// .agents/ and .factory/ artifacts "produced by gen-skill-docs.test.ts" —
// an inter-test ordering dependency that failed with ENOENT on a clean
// clone or when this file ran in isolation. Severed: this describe
// UNCONDITIONALLY renders both hosts into its own --out-dir in beforeAll
// and reads its goldens only from that render — no when-missing check, no
// live-tree reads for the gitignored artifacts, no dependence on what any
// other test left on disk. Comparing a FRESH render to the golden is also
// strictly deterministic: a stale on-disk artifact can no longer mask (or
// fake) a generator regression.
const GOLDEN_OUT = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-golden-out-'));
beforeAll(() => {
const hostArtifacts: Array<[string, string]> = [
['codex', path.join(ROOT, '.agents', 'skills', 'gstack-ship', 'SKILL.md')],
['factory', path.join(ROOT, '.factory', 'skills', 'gstack-ship', 'SKILL.md')],
];
for (const [host, artifact] of hostArtifacts) {
if (fs.existsSync(artifact)) continue;
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', host], {
cwd: ROOT,
});
for (const host of ['codex', 'factory']) {
const result = Bun.spawnSync(
['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', host, '--out-dir', GOLDEN_OUT],
{ cwd: ROOT },
);
if (result.exitCode !== 0) {
throw new Error(
`golden-file beforeAll: gen-skill-docs --host ${host} failed (exit ${result.exitCode}):\n`
`golden-file beforeAll: gen-skill-docs --host ${host} --out-dir failed (exit ${result.exitCode}):\n`
+ result.stderr.toString(),
);
}
}
});
afterAll(() => {
fs.rmSync(GOLDEN_OUT, { recursive: true, force: true });
});
test('Claude ship skill matches golden baseline', () => {
// Deliberately reads the TRACKED ship/SKILL.md (a read, not a write):
// the claude golden pins the committed render. Freshness of the tracked
// tree vs the templates is enforced by gen-skill-docs.test.ts. (An
// out-dir claude render would NOT byte-match this golden — --out-dir
// repoints section-base paths into the render by design.)
const golden = fs.readFileSync(path.join(GOLDEN_DIR, 'claude-ship-SKILL.md'), 'utf-8');
const current = fs.readFileSync(path.join(ROOT, 'ship', 'SKILL.md'), 'utf-8');
expect(current).toBe(golden);
@@ -463,13 +473,13 @@ describe('golden-file regression', () => {
test('Codex ship skill matches golden baseline', () => {
const golden = fs.readFileSync(path.join(GOLDEN_DIR, 'codex-ship-SKILL.md'), 'utf-8');
const current = fs.readFileSync(path.join(ROOT, '.agents', 'skills', 'gstack-ship', 'SKILL.md'), 'utf-8');
const current = fs.readFileSync(path.join(GOLDEN_OUT, '.agents', 'skills', 'gstack-ship', 'SKILL.md'), 'utf-8');
expect(current).toBe(golden);
});
test('Factory ship skill matches golden baseline', () => {
const golden = fs.readFileSync(path.join(GOLDEN_DIR, 'factory-ship-SKILL.md'), 'utf-8');
const current = fs.readFileSync(path.join(ROOT, '.factory', 'skills', 'gstack-ship', 'SKILL.md'), 'utf-8');
const current = fs.readFileSync(path.join(GOLDEN_OUT, '.factory', 'skills', 'gstack-ship', 'SKILL.md'), 'utf-8');
expect(current).toBe(golden);
});
});
+2 -1
View File
@@ -12,6 +12,7 @@
*/
import { expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { judgeRecommendation } from './helpers/llm-judge';
import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers';
@@ -181,5 +182,5 @@ Net: ...`);
`[hedge:${label}] expected commits=false; got ${score.commits}. text="${text}"`,
).toBe(false);
}
}, 240_000);
}, CAPTURE_MS);
});
+80
View File
@@ -0,0 +1,80 @@
/**
* No paid-gated test file may sit outside PAID_TEST_GLOBS.
*
* The orphan class this kills (found 2026-08): a file whose source gates on
* EVALS/tier (so the free suite loads it as describe.skip) but whose NAME
* doesn't match the paid globs (so no paid lane ever selects it) can never
* execute anywhere — forever, silently. Four files were in that state
* (codex-e2e-plan-format, codex-e2e-recommendation-substance,
* llm-judge-recommendation, carve-section-loading), and the tripwire built
* for the adjacent class (test/evals-workflow-matrix.test.ts) couldn't see
* them because it filters on isPaidTestFile() FIRST.
*
* Detection is over source text, so meta-tests and helpers that mention the
* gate patterns need reasoned exemptions (same convention as
* test/egress-receipt-wiring.test.ts's SCANNER_EXEMPT).
*/
import { describe, expect, test } from 'bun:test';
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { isPaidTestFile } from './helpers/paid-test-set';
const ROOT = path.resolve(__dirname, '..');
/** Files that legitimately mention gate patterns without being paid tests. */
const SCANNER_EXEMPT = new Map<string, string>([
// The gate helpers themselves and their free unit tests:
['test/helpers/e2e-gate.ts', 'defines the gate predicates'],
// Meta-tests that quote gate-pattern strings to test classification:
['test/helpers/e2e-gate.unit.test.ts', 'free unit test OF the gate predicates (env stubbed)'],
['test/paid-shards.test.ts', 'quotes tier-guard strings as classification fixtures'],
['test/evals-workflow-matrix.test.ts', 'parses tier guards out of matrix files'],
['test/e2e-tier-alignment.test.ts', 'parses tier guards to enforce alignment'],
['test/paid-orphan-tripwire.test.ts', 'this scanner'],
]);
/**
* Source shapes that mean "this file self-gates on the paid env":
* the shared helpers, or a direct EVALS/EVALS_TIER env read.
*/
const GATE_PATTERNS = [
/\bdescribeE2ETier\s*\(/,
/\be2eTierEnabled\s*\(/,
/process\.env\.EVALS\b/,
];
function trackedTestFiles(): string[] {
const out = spawnSync('git', ['ls-files', '*.test.ts'], { cwd: ROOT, encoding: 'utf-8' });
if (out.status !== 0) throw new Error(`git ls-files failed: ${out.stderr}`);
return out.stdout.split('\n').filter(Boolean);
}
describe('paid orphan tripwire', () => {
test('every EVALS/tier-gated test file is inside PAID_TEST_GLOBS (or exempt with a reason)', () => {
const files = trackedTestFiles();
expect(files.length).toBeGreaterThan(100); // scan-rot guard
const orphans: string[] = [];
for (const rel of files) {
if (isPaidTestFile(rel)) continue;
if (SCANNER_EXEMPT.has(rel)) continue;
const source = fs.readFileSync(path.join(ROOT, rel), 'utf-8');
const hit = GATE_PATTERNS.find((p) => p.test(source));
if (hit) orphans.push(`${rel} (matches ${hit})`);
}
expect(orphans,
'paid-gated test files OUTSIDE the paid globs can never run in any lane. '
+ 'Fix: extend PAID_TEST_GLOBS in test/helpers/paid-test-set.ts (and mirror '
+ 'package.json), or add a reasoned SCANNER_EXEMPT entry if the file only '
+ `mentions the patterns:\n${orphans.join('\n')}`,
).toEqual([]);
});
test('exemption entries stay real (stale entries must be deleted)', () => {
for (const [rel] of SCANNER_EXEMPT) {
expect(fs.existsSync(path.join(ROOT, rel)), `stale SCANNER_EXEMPT entry: ${rel}`).toBe(true);
}
});
});
+190
View File
@@ -0,0 +1,190 @@
/**
* Planner/executor/report contract for the re-platformed paid CI lane.
*
* The classes these pin (each was a live CI failure mode of the old
* hand-enumerated matrix, or a review-identified risk of the migration):
* - per-slice selector divergence → ONE planner manifest, executors consume
* - hollow lanes → a slice with no artifact is a FAILURE, not an absence
* - hollow shards → EVALS_ALL + exit 0 + zero executed tests ≠ pass
* - retry parity → the old matrix rows' earned `retries: 2` survive as a
* literals map, not folklore
*/
import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
import {
applyHollowShardGuard,
buildPaidShardArgs,
buildRunManifest,
parseRunManifest,
retriesForFiles,
RETRY_OVERRIDES,
summarize,
summaryExitCode,
verifySliceResults,
type PaidRunManifest,
type ShardOutcome,
type SliceResult,
} from '../scripts/test-paid-shards';
const ROOT = path.resolve(__dirname, '..');
const outcome = (over: Partial<ShardOutcome>): ShardOutcome => ({
shard: 1,
files: ['test/skill-e2e-x.test.ts'],
status: 'passed',
exitCode: 0,
elapsedMs: 1000,
groupPid: null,
executedTests: 3,
...over,
});
describe('run manifest (planner)', () => {
test('live build: every paid file appears exactly once; planned slices partition 1..K', () => {
const manifest = buildRunManifest({ tier: 'gate', sliceCount: 5, evalsAll: true, env: { EVALS_ALL: '1' } });
const files = manifest.entries.map((e) => e.file);
expect(new Set(files).size).toBe(files.length);
const planned = manifest.entries.filter((e) => e.status === 'planned');
expect(planned.length).toBeGreaterThan(20); // census sanity
for (const entry of planned) {
expect(entry.slice).toBeGreaterThanOrEqual(1);
expect(entry.slice).toBeLessThanOrEqual(5);
}
// Round-robin balance: slice sizes differ by at most 1.
const sizes = [1, 2, 3, 4, 5].map((i) => planned.filter((e) => e.slice === i).length);
expect(Math.max(...sizes) - Math.min(...sizes)).toBeLessThanOrEqual(1);
// Non-runnable entries carry slice 0 and a reason.
for (const entry of manifest.entries.filter((e) => e.status !== 'planned')) {
expect(entry.slice).toBe(0);
expect(entry.reason ?? '').not.toBe('');
}
});
test('deterministic for identical inputs', () => {
const opts = { tier: 'periodic' as const, sliceCount: 4, evalsAll: true, env: { EVALS_ALL: '1' } };
expect(buildRunManifest(opts)).toEqual(buildRunManifest(opts));
});
test('parse round-trips and rejects malformed manifests', () => {
// EVALS_ALL short-circuits diff selection BEFORE any git walk: selection
// is deliberately fail-closed on git errors, and CI's shallow free-tests
// checkout has no base ref (first CI run failed here with
// "ambiguous argument 'main...HEAD'").
const manifest = buildRunManifest({ tier: 'gate', sliceCount: 2, evalsAll: false, env: { EVALS_ALL: '1' } });
expect(parseRunManifest(JSON.stringify(manifest))).toEqual(manifest);
expect(() => parseRunManifest('{}')).toThrow(/version/);
expect(() => parseRunManifest(JSON.stringify({ ...manifest, tier: 'e2e' }))).toThrow(/tier/);
expect(() => parseRunManifest(JSON.stringify({ ...manifest, sliceCount: 0 }))).toThrow(/sliceCount/);
const outOfRange = {
...manifest,
entries: [{ file: 'test/skill-e2e-x.test.ts', slice: 9, status: 'planned' }],
};
expect(() => parseRunManifest(JSON.stringify(outOfRange))).toThrow(/out-of-range/);
});
});
describe('slice-result reconciliation (report)', () => {
const manifest: PaidRunManifest = {
version: 1,
tier: 'gate',
evalsAll: false,
sliceCount: 2,
selectionReason: 'test fixture',
entries: [
{ file: 'test/skill-e2e-a.test.ts', slice: 1, status: 'planned' },
{ file: 'test/skill-e2e-b.test.ts', slice: 2, status: 'planned' },
{ file: 'test/skill-e2e-c.test.ts', slice: 0, status: 'skipped-by-diff', reason: 'unselected' },
],
};
const slice = (index: number, files: string[], status: ShardOutcome['status'] = 'passed'): SliceResult => ({
version: 1,
tier: 'gate',
sliceIndex: index,
sliceCount: 2,
outcomes: files.map((f) => ({ files: [f], status, exitCode: 0, elapsedMs: 5, executedTests: 2 })),
});
test('all slices present and passing → ok', () => {
const verdict = verifySliceResults(manifest, [
slice(1, ['test/skill-e2e-a.test.ts']),
slice(2, ['test/skill-e2e-b.test.ts']),
]);
expect(verdict).toEqual({ ok: true, problems: [] });
});
test('a missing slice artifact is a FAILURE, not an absence', () => {
const verdict = verifySliceResults(manifest, [slice(1, ['test/skill-e2e-a.test.ts'])]);
expect(verdict.ok).toBe(false);
expect(verdict.problems.join('\n')).toContain('slice 2/2 reported NO result');
});
test('a planned shard nobody reported fails even when its slice reported', () => {
const verdict = verifySliceResults(manifest, [
slice(1, []),
slice(2, ['test/skill-e2e-b.test.ts']),
]);
expect(verdict.ok).toBe(false);
expect(verdict.problems.join('\n')).toContain('never reported');
});
test('wrong-slice, duplicate, cross-tier, and failing outcomes all surface', () => {
const wrongSlice = verifySliceResults(manifest, [
slice(1, ['test/skill-e2e-b.test.ts']),
slice(2, ['test/skill-e2e-a.test.ts']),
]);
expect(wrongSlice.ok).toBe(false);
const failing = verifySliceResults(manifest, [
slice(1, ['test/skill-e2e-a.test.ts'], 'failed'),
slice(2, ['test/skill-e2e-b.test.ts']),
]);
expect(failing.problems.join('\n')).toContain('test/skill-e2e-a.test.ts: failed');
const crossTier = verifySliceResults(manifest, [
{ ...slice(1, ['test/skill-e2e-a.test.ts']), tier: 'periodic' },
slice(2, ['test/skill-e2e-b.test.ts']),
]);
expect(crossTier.problems.join('\n')).toContain('ran tier periodic');
});
});
describe('hollow-shard guard', () => {
test('EVALS_ALL: passed with 0 executed tests becomes passed-empty and fails the run', () => {
const guarded = applyHollowShardGuard([outcome({ executedTests: 0 })], { evalsAll: true, warn: () => {} });
expect(guarded[0].status).toBe('passed-empty');
const summary = summarize(guarded);
expect(summary.failed).toBe(1);
expect(summaryExitCode(summary)).toBe(1);
});
test('selective run: same shape stays passed, warns once', () => {
const warnings: string[] = [];
const guarded = applyHollowShardGuard([outcome({ executedTests: 0 })], {
evalsAll: false, warn: (line) => warnings.push(line),
});
expect(guarded[0].status).toBe('passed');
expect(warnings).toHaveLength(1);
});
test('unknown executedTests (null) is never guessed hollow', () => {
const guarded = applyHollowShardGuard([outcome({ executedTests: null })], { evalsAll: true });
expect(guarded[0].status).toBe('passed');
});
});
describe('retry parity', () => {
test('overrides exist only for the files whose matrix rows earned them, and each names a real file', () => {
expect(Object.keys(RETRY_OVERRIDES).sort()).toEqual([
'test/skill-e2e-office-hours-auto-mode.test.ts',
'test/skill-e2e-plan-mode-no-op.test.ts',
'test/skill-e2e-workflow.test.ts',
]);
for (const file of Object.keys(RETRY_OVERRIDES)) {
expect(fs.existsSync(path.join(ROOT, file)), `stale RETRY_OVERRIDES entry: ${file}`).toBe(true);
}
expect(retriesForFiles(['test/skill-e2e-workflow.test.ts'])).toBe(2);
expect(retriesForFiles(['test/skill-e2e-retro.test.ts'])).toBe(1);
expect(buildPaidShardArgs(['x'], 1000, 4, 2)).toContain('2');
expect(buildPaidShardArgs(['x'], 1000, 4).join(' ')).toContain('--retry 1');
});
});
+117
View File
@@ -0,0 +1,117 @@
/**
* Parent/child selection-drift pins for EVALS_SELECTION_JSON.
*
* The sharded paid runner computes the diff selection ONCE in the parent
* (computePaidDiffSelection in scripts/test-paid-shards.ts), serializes it
* (serializePaidDiffSelection) into every shard child's env, and
* test/helpers/e2e-helpers.ts adopts it at module load (parseEvalsSelectionJson
* via resolveModuleSelection) instead of re-deriving it per shard — which,
* whenever touchfiles-data.ts was in the diff, spawned one bun subprocess PER
* CHILD to evaluate the old data file (test-selection.ts map-diff path).
*
* These pins hold the two sides to IDENTICAL selection decisions across the
* serialize/parse boundary, and the child to fail-open (local recompute with
* one stderr warning) on any parse/shape failure.
*/
import { describe, test, expect } from 'bun:test';
import {
computePaidDiffSelection,
serializePaidDiffSelection,
type PaidDiffSelection,
} from '../scripts/test-paid-shards';
import { parseEvalsSelectionJson, resolveModuleSelection } from './helpers/e2e-helpers';
/** The parent's per-test decision shape (PaidDiffSelection.selectedNames). */
const parentWouldRun = (selection: PaidDiffSelection, name: string): boolean =>
selection.selectedNames === null || selection.selectedNames.has(name);
/** The child's per-test decision shape (testIfSelected / describeIfSelected). */
const childWouldRun = (selected: string[] | null, name: string): boolean =>
selected === null || selected.includes(name);
const NAMES = ['qa-workflow', 'review-army', 'ship-docsync', 'unmapped-test'];
describe('EVALS_SELECTION_JSON parent -> child propagation', () => {
test('a concrete selection round-trips to identical decisions', () => {
const fixture: PaidDiffSelection = {
selectedNames: new Set(['qa-workflow', 'ship-docsync']),
reason: 'diff',
totalTests: 4,
};
const parsed = parseEvalsSelectionJson(serializePaidDiffSelection(fixture));
expect(parsed.selected).toEqual(['qa-workflow', 'ship-docsync']);
expect(parsed.reason).toBe('diff');
for (const name of NAMES) {
expect(childWouldRun(parsed.selected, name), name).toBe(parentWouldRun(fixture, name));
}
});
test('run-all (null) round-trips to null — child runs everything', () => {
// computePaidDiffSelection is the REAL parent function; EVALS_ALL is its
// git-free path, so the serializer sees input exactly as produced.
const selection = computePaidDiffSelection({ EVALS_ALL: '1' } as NodeJS.ProcessEnv);
expect(selection.selectedNames).toBeNull();
const parsed = parseEvalsSelectionJson(serializePaidDiffSelection(selection));
expect(parsed.selected).toBeNull();
for (const name of NAMES) {
expect(childWouldRun(parsed.selected, name)).toBe(parentWouldRun(selection, name));
}
});
test('empty selection stays empty — nothing selected is NOT run-all', () => {
const fixture: PaidDiffSelection = { selectedNames: new Set(), reason: 'diff', totalTests: 4 };
const parsed = parseEvalsSelectionJson(serializePaidDiffSelection(fixture));
expect(parsed.selected).toEqual([]);
for (const name of NAMES) {
expect(childWouldRun(parsed.selected, name)).toBe(false);
expect(parentWouldRun(fixture, name)).toBe(false);
}
});
test('parser THROWS on malformed JSON and wrong shapes', () => {
expect(() => parseEvalsSelectionJson('{"selected": ')).toThrow();
expect(() => parseEvalsSelectionJson('null')).toThrow();
expect(() => parseEvalsSelectionJson('[1,2]')).toThrow();
expect(() => parseEvalsSelectionJson('{"selected": 42}')).toThrow();
expect(() => parseEvalsSelectionJson('{"selected": ["a", 7]}')).toThrow();
});
test('malformed EVALS_SELECTION_JSON falls back to local compute with one stderr warning', () => {
const warnings: string[] = [];
let computed = 0;
const result = resolveModuleSelection(
'{"selected": 42}',
() => { computed += 1; return ['locally-computed']; },
(text) => warnings.push(text),
);
expect(result).toEqual(['locally-computed']); // fail-open preserved
expect(computed).toBe(1);
expect(warnings.length).toBe(1);
expect(warnings[0]).toContain('EVALS_SELECTION_JSON');
});
test('absent env var computes locally, silently (non-sharded entrypoints unchanged)', () => {
const writes: string[] = [];
let computed = 0;
const result = resolveModuleSelection(
undefined,
() => { computed += 1; return null; },
(text) => writes.push(text),
);
expect(result).toBeNull();
expect(computed).toBe(1);
expect(writes.length).toBe(0);
});
test('a valid env var short-circuits local derivation entirely', () => {
let computed = 0;
const result = resolveModuleSelection(
serializePaidDiffSelection({ selectedNames: new Set(['a']), reason: 'diff', totalTests: 1 }),
() => { computed += 1; return null; },
() => {},
);
expect(result).toEqual(['a']);
expect(computed).toBe(0); // no git walk, no map-diff bun subprocess
});
});
+60 -3
View File
@@ -11,6 +11,7 @@
import { describe, test, expect } from 'bun:test';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
const ROOT = path.resolve(import.meta.dir, '..');
@@ -43,15 +44,22 @@ describe('paid test enumeration', () => {
// kept here as a regression pin: its glob-invisibility is exactly how
// two gate tests went unexecuted for ~8 releases before the rehoming.
expect(isPaidTestFile('test/skill-e2e.test.ts')).toBe(false);
expect(isPaidTestFile('test/codex-e2e-recommendation-substance.test.ts')).toBe(false);
expect(isPaidTestFile('test/paid-shards.test.ts')).toBe(false);
// The 2026-08 orphan fix: these four were API-spending files OUTSIDE the
// globs — self-skipping in the free suite and absent from the paid
// census, so they could never run in any lane.
expect(isPaidTestFile('test/codex-e2e-recommendation-substance.test.ts')).toBe(true);
expect(isPaidTestFile('test/codex-e2e-plan-format.test.ts')).toBe(true);
expect(isPaidTestFile('test/llm-judge-recommendation.test.ts')).toBe(true);
expect(isPaidTestFile('test/carve-section-loading.test.ts')).toBe(true);
expect(isPaidTestFile('test/skill-llm-eval-spec.test.ts')).toBe(true);
});
test('discovers files and gives each one its own shard', () => {
const files = collectPaidTestFiles();
expect(files.length).toBeGreaterThan(0);
expect(files.every(isPaidTestFile)).toBe(true);
expect(PAID_TEST_GLOBS.length).toBe(6);
expect(PAID_TEST_GLOBS.length).toBe(7);
const shards = planPaidShards(files);
expect(shards.flat().sort()).toEqual([...files].sort());
@@ -108,10 +116,17 @@ describe('tier classification', () => {
describe('shard execution', () => {
const BUSY_LOOP = 'const end = Date.now() + 600000; while (Date.now() < end) {}';
// PIN UPDATE (deliberate): the strict expectedFiles check is now enforced
// for injected fake commands too (drift fix toward the free runner's
// behavior), so a fake PASSING command must print a synthetic bun terminal
// summary — a summary-less exit 0 is the truncation class and reads FAILED.
const PASS_WITH_SUMMARY = 'console.log("ok"); console.log("Ran 1 tests across 1 files. [1ms]")';
const commandFor = (files: string[]) => {
if (files[0] === 'spin') return { command: process.execPath, args: ['-e', BUSY_LOOP] };
if (files[0] === 'fail') return { command: process.execPath, args: ['-e', 'process.exit(3)'] };
return { command: process.execPath, args: ['-e', 'console.log("ok")'] };
if (files[0] === 'silent-pass') return { command: process.execPath, args: ['-e', 'console.log("ok")'] };
return { command: process.execPath, args: ['-e', PASS_WITH_SUMMARY] };
};
test('a spinning shard times out, is killed, and the run continues', async () => {
@@ -146,6 +161,48 @@ describe('shard execution', () => {
expect(lines.some((l) => /PASSED in \d+s/.test(l))).toBe(true);
}, 30_000);
test('exit 0 WITHOUT the terminal summary is FAILED — enforced for injected commands too', async () => {
// The invisible-non-execution backstop: previously the paid runner
// exempted injected commandFor from the expectedFiles check, so a fake
// that exited 0 without bun's terminal summary recorded 'passed'. Now it
// matches the free runner: enforcement always on.
const summary = await runPaidShards([['silent-pass']], {
timeoutMs: 30_000, jobs: 1, commandFor, log: () => {},
});
expect(summary.outcomes[0].status).toBe('failed');
}, 30_000);
test('shard output spools to a per-shard log file; failures name the path', async () => {
const logDir = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-shard-logs-'));
const lines: string[] = [];
try {
const summary = await runPaidShards([['fail'], ['pass']], {
timeoutMs: 30_000, jobs: 2, commandFor, logDir, log: (line) => lines.push(line),
});
const byName = (name: string) => summary.outcomes.find((o) => o.files[0] === name) as ShardOutcome;
expect(byName('fail').status).toBe('failed');
expect(byName('pass').status).toBe('passed');
// One log per shard, named by slug, and it holds the child's full stream
// (nothing buffered in RAM: the file IS the record).
const logs = fs.readdirSync(logDir).sort();
expect(logs.length).toBe(2);
expect(logs.some((f) => f.includes('fail'))).toBe(true);
const passLog = logs.find((f) => f.includes('pass')) as string;
expect(fs.readFileSync(path.join(logDir, passLog), 'utf8')).toContain('Ran 1 tests across 1 files.');
// Every shard announces its log path up front; the FAILED terminal line
// repeats it, the PASSED one stays clean.
expect(lines.filter((l) => l.includes('full log:') && !l.includes('FAILED')).length).toBe(2);
const failLine = lines.find((l) => l.includes('FAILED')) as string;
expect(failLine).toContain(logDir);
const passLine = lines.find((l) => l.includes('PASSED')) as string;
expect(passLine).not.toContain(logDir);
} finally {
fs.rmSync(logDir, { recursive: true, force: true });
}
}, 30_000);
test('summarize reports shards that never ran', () => {
const summary = summarize([
{ shard: 1, files: ['a'], status: 'passed', exitCode: 0, elapsedMs: 1, groupPid: 1 },
+45
View File
@@ -0,0 +1,45 @@
/**
* The periodic exclude list is a set of DECISIONS, not a place tests go to
* die: every entry names a real file (a deleted/renamed file must drop its
* entry) and carries a non-empty reason + tracking pointer (the re-entry
* condition lives there). The runner surfaces each exclusion per run, and
* removing an entry re-activates the file on the next weekly lane.
*/
import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { PERIODIC_CI_EXCLUDE } from './helpers/periodic-exclude-data';
import { isPaidTestFile } from './helpers/paid-test-set';
import { selectPaidTestFiles } from '../scripts/test-paid-shards';
const ROOT = path.resolve(__dirname, '..');
describe('periodic exclude policy', () => {
test('every entry names a real paid file and carries reason + tracking', () => {
const entries = Object.entries(PERIODIC_CI_EXCLUDE);
expect(entries.length).toBeGreaterThan(0);
for (const [file, meta] of entries) {
expect(fs.existsSync(path.join(ROOT, file)), `stale exclude entry: ${file}`).toBe(true);
expect(isPaidTestFile(file), `${file} is not a paid file — exclusion is meaningless`).toBe(true);
expect(meta.reason.length, `${file}: empty reason`).toBeGreaterThan(20);
expect(meta.tracking.length, `${file}: empty tracking pointer`).toBeGreaterThan(5);
}
});
test('exclusions apply to the periodic tier only, with the reason surfaced', () => {
const files = Object.keys(PERIODIC_CI_EXCLUDE);
const periodic = selectPaidTestFiles(files, 'periodic');
expect(periodic.selected).toEqual([]);
for (const { reason } of periodic.excluded) {
expect(reason).toStartWith('excluded: ');
expect(reason).toContain('[');
}
// Gate tier ignores the list (these files are periodic-tier anyway; the
// list must never leak into gate semantics).
const gate = selectPaidTestFiles(files, 'gate');
for (const { reason } of gate.excluded) {
expect(reason).not.toStartWith('excluded: ');
}
});
});
+29
View File
@@ -182,6 +182,35 @@ describe('gstack-wtree', () => {
});
});
test('racy-git window: a same-size rewrite pinned to the index timestamp changes the fingerprint', () => {
withScratchRepo((repoDir, wtree) => {
const file = path.join(repoDir, 'a.txt');
const indexPath = path.join(repoDir, '.git', 'index');
// ctime can't be restored after a rewrite; production hits this window
// when everything lands in the same second (ctime SECONDS match).
// trustctime=false isolates the racy mechanism deterministically
// instead of racing a second boundary.
gitIn(repoDir, 'config core.trustctime false');
// Pin the cached entry's mtime to a fixed timestamp (zero nsec, so the
// restore below is exact even on USE_NSEC git builds).
const pinned = new Date('2026-01-01T12:00:00Z');
fs.utimesSync(file, pinned, pinned);
gitIn(repoDir, 'add a.txt');
const clean = wtree();
// Same-size rewrite restored to the pinned stat, with the index file
// itself pinned to the SAME timestamp: the entry is stat-identical to
// its stale cache and sits exactly on git's racy-git boundary.
// gstack-wtree must carry the real index's mtime onto its temp copy —
// a fresh-stamped copy marks the entry non-racy, trusts the stale stat
// cache, and the edit vanishes from the fingerprint (evidence would
// stay FRESH after a source change).
fs.writeFileSync(file, 'howdy\n'); // same byte length as 'hello\n'
fs.utimesSync(file, pinned, pinned);
fs.utimesSync(indexPath, pinned, pinned);
expect(wtree()).not.toBe(clean);
});
});
test('exits non-zero outside a git repo', () => {
const nonGit = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-wtree-nongit-'));
try {
+91
View File
@@ -0,0 +1,91 @@
/**
* Direct pins for runShardChild (scripts/test-strict-output.ts) the shared
* spawn/detached/group-kill/wall-timer/reap lifecycle extracted from the paid
* runner's runPaidShard, designed for scripts/test-free-shards.ts to migrate
* onto next. test/paid-shards.test.ts pins the paid runner end-to-end; these
* pin the helper's own contract so the free-runner migration has a floor.
*/
import { describe, test, expect } from 'bun:test';
import * as os from 'os';
import * as path from 'path';
import type { ChildProcess } from 'child_process';
import { runShardChild } from '../scripts/test-strict-output';
/** Collect the child's full stdout+stderr, resolving only when drained. */
function collectingHook(chunks: string[]) {
return (child: ChildProcess): Array<Promise<void>> => {
const consume = (stream: NodeJS.ReadableStream | null): Promise<void> =>
stream
? new Promise((resolve, reject) => {
stream.on('data', (chunk: Buffer | string) => chunks.push(chunk.toString()));
stream.on('end', resolve);
stream.on('error', reject);
})
: Promise.resolve();
return [consume(child.stdout), consume(child.stderr)];
};
}
describe('runShardChild', () => {
test('clean exit: exitCode 0, not timed out, output drained before resolve', async () => {
const chunks: string[] = [];
const result = await runShardChild({
command: process.execPath,
args: ['-e', 'console.log("hello-from-child")'],
cwd: process.cwd(),
env: process.env,
timeoutMs: 30_000,
hookStreams: collectingHook(chunks),
});
expect(result.exitCode).toBe(0);
expect(result.timedOut).toBe(false);
expect(result.groupPid).toBeGreaterThan(0);
// The hookStreams promises are awaited AFTER close — trailing output is
// fully drained before callers read their classifier/log state.
expect(chunks.join('')).toContain('hello-from-child');
}, 30_000);
test('non-zero exit code propagates untouched', async () => {
const result = await runShardChild({
command: process.execPath,
args: ['-e', 'process.exit(7)'],
cwd: process.cwd(),
env: process.env,
timeoutMs: 30_000,
hookStreams: () => [],
});
expect(result.exitCode).toBe(7);
expect(result.timedOut).toBe(false);
}, 30_000);
test('a spinning child is group-SIGKILLed at the wall deadline and reported timedOut', async () => {
const startedAt = Date.now();
const result = await runShardChild({
command: process.execPath,
// A real busy loop: an in-process timer could never fire in this child.
args: ['-e', 'const end = Date.now() + 600000; while (Date.now() < end) {}'],
cwd: process.cwd(),
env: process.env,
timeoutMs: 1_200,
hookStreams: () => [],
});
expect(result.timedOut).toBe(true);
expect(Date.now() - startedAt).toBeLessThan(30_000);
if (process.platform !== 'win32') {
// The whole group is gone, not left to burn a core.
expect(() => process.kill(result.groupPid as number, 0)).toThrow();
}
}, 30_000);
test('a spawn failure THROWS so callers keep their could-not-run handling', async () => {
await expect(runShardChild({
command: path.join(os.tmpdir(), 'definitely-not-a-real-binary-8b1f'),
args: [],
cwd: process.cwd(),
env: process.env,
timeoutMs: 5_000,
hookStreams: () => [],
})).rejects.toThrow();
}, 30_000);
});
@@ -23,6 +23,7 @@
* A/B and matrix evals (test/helpers/auq-sdk-capture.ts).
*/
import { test, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import {
@@ -86,6 +87,6 @@ describeE2E('AskUserQuestion format compliance (gate)', () => {
);
}
},
300_000,
CAPTURE_MS,
);
});
+2 -1
View File
@@ -16,6 +16,7 @@
* (N SDK runs, ~$0.50-1 each).
*/
import { test } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import {
@@ -99,6 +100,6 @@ describeE2E('AUQ consistency across runs (periodic)', () => {
`format elements every run; substance ${minSub}-${maxSub}`,
);
},
N_RUNS * 300_000 + 60_000,
N_RUNS * CAPTURE_MS + 60_000,
);
});
+2 -1
View File
@@ -23,6 +23,7 @@
* Run a subset in the foreground with AUQ_MATRIX_ONLY="plan-eng-review,cso".
*/
import { test } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import {
@@ -174,7 +175,7 @@ describeE2E('AUQ behavioral matrix (periodic)', () => {
);
}
},
300_000,
CAPTURE_MS,
);
}
});
@@ -23,6 +23,7 @@
* strictly less unrelated review-section text in context.
*/
import { test } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import {
@@ -109,6 +110,6 @@ describeE2E('AUQ no-degradation: verbose vs carved (periodic)', () => {
// eslint-disable-next-line no-console
console.log('[AUQ-AB] NO DEGRADATION:\n' + summary);
},
600_000,
CAPTURE_LONG_MS,
);
});
+5 -2
View File
@@ -38,6 +38,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
import * as fs from 'fs';
@@ -67,6 +68,8 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
// claude would resolve). The preference file path keys on this slug.
const slugBin = path.join(ROOT, 'bin', 'gstack-slug');
const slugRes = spawnSync(slugBin, [], {
// LIVE-REPO CWD: gstack-slug resolves the slug from this repo's git
// remote — must match what the spawned claude (repo cwd) resolves.
cwd: ROOT,
env: { ...process.env, GSTACK_HOME: tmpHome },
encoding: 'utf-8',
@@ -111,7 +114,7 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
skillName: 'plan-ceo-review',
inPlanMode: true,
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
timeoutMs: 540_000,
timeoutMs: CAPTURE_LONG_MS,
env: { GSTACK_HOME: tmpHome, CONDUCTOR_WORKSPACE_PATH: tmpHome },
});
@@ -135,5 +138,5 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
} finally {
try { fs.rmSync(tmpHome, { recursive: true, force: true }); } catch { /* best-effort */ }
}
}, 660_000);
}, PTY_MS);
});
+2 -1
View File
@@ -26,6 +26,7 @@
*/
import { test, expect } from 'bun:test';
import { PTY_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
@@ -184,6 +185,6 @@ describeE2E('/autoplan chain ordering (periodic)', () => {
try { fs.rmSync(tempDir, { recursive: true, force: true }); } catch { /* ignore */ }
}
},
1_200_000, // 20 min absolute test ceiling
PTY_LONG_MS, // 20 min absolute test ceiling
);
});
+2 -1
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId, evalsEnabled,
@@ -98,7 +99,7 @@ Add a new /greet skill that prints a welcome message.
testName: 'autoplan-dual-voice',
workingDirectory: workDir,
prompt: `/autoplan ${planPath}`,
timeout: 600_000, // 10 min
timeout: CAPTURE_LONG_MS, // 10 min
// /autoplan spawns subagents and calls codex via Bash; it needs the
// full tool set to get past Phase 1. Bash+Read+Write alone wasn't
// enough — the skill stalled trying to invoke Agent/Skill.
+9 -8
View File
@@ -19,6 +19,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { ClaudeAdapter } from './helpers/providers/claude';
import { GptAdapter } from './helpers/providers/gpt';
import { GeminiAdapter } from './helpers/providers/gemini';
@@ -94,7 +95,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
process.stderr.write(`\nclaude live smoke: SKIPPED — ${check.reason}\n`);
return;
}
const result = await claude.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 });
const result = await claude.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS });
if (result.error) {
throw new Error(`claude errored: ${result.error.code}${result.error.reason}`);
}
@@ -106,7 +107,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
expect(result.modelUsed.length).toBeGreaterThan(0);
const cost = claude.estimateCost(result.tokens, result.modelUsed);
expect(cost).toBeGreaterThan(0);
}, 150_000);
}, CAPTURE_MS);
test('gpt: trivial prompt produces parseable output', async () => {
const check = await gpt.available();
@@ -114,7 +115,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
process.stderr.write(`\ngpt live smoke: SKIPPED — ${check.reason}\n`);
return;
}
const result = await gpt.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 });
const result = await gpt.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS });
if (result.error) {
throw new Error(`gpt errored: ${result.error.code}${result.error.reason}`);
}
@@ -125,7 +126,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
expect(typeof result.modelUsed).toBe('string');
const cost = gpt.estimateCost(result.tokens, result.modelUsed);
expect(cost).toBeGreaterThan(0);
}, 150_000);
}, CAPTURE_MS);
test('gemini: trivial prompt produces parseable output', async () => {
const check = await gemini.available();
@@ -133,7 +134,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
process.stderr.write(`\ngemini live smoke: SKIPPED — ${check.reason}\n`);
return;
}
const result = await gemini.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 });
const result = await gemini.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS });
if (result.error) {
// auth / rate_limit are ENVIRONMENT conditions the test can't act on
// (e.g. Google deprecated the individual code-assist auth path — the
@@ -155,7 +156,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
expect(result.durationMs).toBeGreaterThan(0);
expect(typeof result.modelUsed).toBe('string');
expect(result.modelUsed.length).toBeGreaterThan(0);
}, 150_000);
}, CAPTURE_MS);
test('timeout error surfaces as error.code=timeout (no exception)', async () => {
// Use whatever adapter is available first — all three should share timeout semantics.
@@ -183,7 +184,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
prompt: PROMPT,
workdir,
providers: ['claude', 'gpt', 'gemini'],
timeoutMs: 120_000,
timeoutMs: JUDGE_MS,
skipUnavailable: false,
});
expect(report.entries).toHaveLength(3);
@@ -201,5 +202,5 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
if (!hadSuccess) {
process.stderr.write('\nrunBenchmark live: no provider produced a clean result (no auth?)\n');
}
}, 300_000);
}, CAPTURE_MS);
});
+3 -2
View File
@@ -21,6 +21,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'fs';
import * as os from 'os';
@@ -150,7 +151,7 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
fs.rmSync(fakeBinDir, { recursive: true, force: true });
fs.rmSync(tempHome, { recursive: true, force: true });
}
}, 180_000);
}, CAPTURE_MS);
test('privacy gate does NOT fire when artifacts_sync_mode_prompted is already true', async () => {
// Same staging, but prompted=true this time. Gate should be silent.
@@ -228,5 +229,5 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
fs.rmSync(fakeBinDir, { recursive: true, force: true });
fs.rmSync(tempHome, { recursive: true, force: true });
}
}, 180_000);
}, CAPTURE_MS);
});
+12 -11
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, browseBin, runId, evalsEnabled,
@@ -47,7 +48,7 @@ describeIfSelected('Skill E2E tests', [
Report the results of each command.`,
workingDirectory: tmpDir,
maxTurns: 7,
timeout: 60_000,
timeout: JUDGE_MS,
testName: 'browse-basic',
runId,
});
@@ -56,7 +57,7 @@ Report the results of each command.`,
recordE2E(evalCollector, 'browse basic commands', 'Skill E2E tests', result);
expect(result.browseErrors).toHaveLength(0);
expect(result.exitReason).toBe('success');
}, 90_000);
}, JUDGE_MS);
testConcurrentIfSelected('browse-snapshot', async () => {
const result = await runSkillTest({
@@ -69,7 +70,7 @@ Report the results of each command.`,
Report what each command returned.`,
workingDirectory: tmpDir,
maxTurns: 9,
timeout: 60_000,
timeout: JUDGE_MS,
testName: 'browse-snapshot',
runId,
});
@@ -81,7 +82,7 @@ Report what each command returned.`,
console.warn('Browse errors (non-fatal):', result.browseErrors);
}
expect(result.exitReason).toBe('success');
}, 90_000);
}, JUDGE_MS);
testConcurrentIfSelected('skillmd-setup-discovery', async () => {
// P2 (v1.2.0): the browse SETUP/binary-discovery block moved from the root
@@ -104,7 +105,7 @@ Then run: $B text
Report whether it worked.`,
workingDirectory: tmpDir,
maxTurns: 10,
timeout: 60_000,
timeout: JUDGE_MS,
testName: 'skillmd-setup-discovery',
runId,
});
@@ -112,7 +113,7 @@ Report whether it worked.`,
recordE2E(evalCollector, 'SKILL.md setup block discovery', 'Skill E2E tests', result);
expect(result.browseErrors).toHaveLength(0);
expect(result.exitReason).toBe('success');
}, 90_000);
}, JUDGE_MS);
testConcurrentIfSelected('skillmd-no-local-binary', async () => {
// Create a tmpdir with no browse binary — no local .claude/skills/gstack/browse/dist/browse
@@ -149,7 +150,7 @@ Report the exact output. Do NOT try to fix or install anything — just report w
// Clean up
try { fs.rmSync(emptyDir, { recursive: true, force: true }); } catch {}
}, 60_000);
}, JUDGE_MS);
testConcurrentIfSelected('skillmd-outside-git', async () => {
// Create a tmpdir outside any git repo
@@ -182,7 +183,7 @@ Report the exact output — either "READY: <path>" or "NEEDS_SETUP".`,
// Clean up
try { fs.rmSync(nonGitDir, { recursive: true, force: true }); } catch {}
}, 60_000);
}, JUDGE_MS);
testConcurrentIfSelected('operational-learning', async () => {
const opDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-oplearn-'));
@@ -286,7 +287,7 @@ Log the operational learning now. Then say what you logged.`,
// Clean up
try { fs.rmSync(opDir, { recursive: true, force: true }); } catch {}
}, 90_000);
}, JUDGE_MS);
testConcurrentIfSelected('session-awareness', async () => {
const sessionDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-session-'));
@@ -353,7 +354,7 @@ Since this is non-interactive, DO NOT actually call AskUserQuestion. Instead, wr
Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple windows and may not remember what this conversation is about. Re-ground them.`,
workingDirectory: sessionDir,
maxTurns: 8,
timeout: 60_000,
timeout: JUDGE_MS,
testName: 'session-awareness',
runId,
});
@@ -394,7 +395,7 @@ Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple wi
// Clean up
try { fs.rmSync(sessionDir, { recursive: true, force: true }); } catch {}
}, 90_000);
}, JUDGE_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
+3 -2
View File
@@ -21,6 +21,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
@@ -46,7 +47,7 @@ describeE2E('Conductor renders decisions as prose (periodic)', () => {
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
env: { CONDUCTOR_WORKSPACE_PATH: '/tmp/conductor-prose-e2e' },
initialPlanContent: FLAWED_PLAN,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
// The decision must reach the human as prose. 'silent_write' (wrote findings
@@ -65,5 +66,5 @@ describeE2E('Conductor renders decisions as prose (periodic)', () => {
}
// A prose-rendered decision brief was observed at some point in the run.
expect(obs.proseAUQEverObserved).toBe(true);
}, 360_000);
}, CAPTURE_LONG_MS);
});
+17 -16
View File
@@ -12,6 +12,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId, evalsEnabled,
@@ -163,7 +164,7 @@ describeIfSelected('Context Skills E2E (live-fire)', [
env: { GSTACK_HOME: gstackHome },
maxTurns: 12,
allowedTools: ['Skill', 'Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-save-routing',
runId,
});
@@ -185,7 +186,7 @@ describeIfSelected('Context Skills E2E (live-fire)', [
expect(routedToContextSave).toBe(true);
expect(files.length).toBeGreaterThan(0);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000);
}, CAPTURE_MS);
// ── 2. Round-trip: save then restore in the same session ─────────────
testConcurrentIfSelected('context-save-then-restore-roundtrip', async () => {
@@ -205,7 +206,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 25,
allowedTools: ['Skill', 'Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'context-save-then-restore-roundtrip',
runId,
});
@@ -232,7 +233,7 @@ Do NOT use AskUserQuestion.`,
expect(files.length).toBeGreaterThan(0);
expect(restoreMentionsTitle).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 240_000);
}, CAPTURE_MS);
// ── 3. /context-restore <fragment> loads the matching save ───────────
testConcurrentIfSelected('context-restore-fragment-match', async () => {
@@ -255,7 +256,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 10,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-restore-fragment-match',
runId,
});
@@ -279,7 +280,7 @@ Do NOT use AskUserQuestion.`,
expect(loadedPayments).toBe(true);
expect(didNotLoadOthers).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000);
}, CAPTURE_MS);
// ── 4. /context-restore with zero saves → graceful empty-state ───────
testConcurrentIfSelected('context-restore-empty-state', async () => {
@@ -294,7 +295,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 8,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'context-restore-empty-state',
runId,
});
@@ -319,7 +320,7 @@ Do NOT use AskUserQuestion.`,
expect(routedToRestore).toBe(true);
expect(gracefulMessage).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 150_000);
}, CAPTURE_MS);
// ── 5. /context-restore list redirects to /context-save list ─────────
testConcurrentIfSelected('context-restore-list-delegates', async () => {
@@ -334,7 +335,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 8,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'context-restore-list-delegates',
runId,
});
@@ -357,7 +358,7 @@ Do NOT use AskUserQuestion.`,
expect(routedToRestore).toBe(true);
expect(mentionsSaveList).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 150_000);
}, CAPTURE_MS);
// ── 6. Legacy compat: pre-rename save files still load ───────────────
testConcurrentIfSelected('context-restore-legacy-compat', async () => {
@@ -381,7 +382,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 8,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-restore-legacy-compat',
runId,
});
@@ -414,7 +415,7 @@ Do NOT use AskUserQuestion.`,
expect(routedToRestore).toBe(true);
expect(loadedLegacy).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000);
}, CAPTURE_MS);
// ── 7. /context-save list: default filters to current branch ─────────
testConcurrentIfSelected('context-save-list-current-branch', async () => {
@@ -437,7 +438,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 10,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-save-list-current-branch',
runId,
});
@@ -472,7 +473,7 @@ Do NOT use AskUserQuestion.`,
expect(hidesAlpha).toBe(true);
expect(hidesBeta).toBe(true);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000);
}, CAPTURE_MS);
// ── 8. /context-save list --all: shows every branch ──────────────────
testConcurrentIfSelected('context-save-list-all-branches', async () => {
@@ -494,7 +495,7 @@ Do NOT use AskUserQuestion.`,
env: { GSTACK_HOME: gstackHome },
maxTurns: 10,
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-save-list-all-branches',
runId,
});
@@ -520,5 +521,5 @@ Do NOT use AskUserQuestion.`,
expect(routed).toBe(true);
expect(filesShown).toBe(3);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, 180_000);
}, CAPTURE_MS);
});
+5 -4
View File
@@ -20,6 +20,7 @@
*/
import { test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId,
@@ -78,7 +79,7 @@ Output the diagram directly.`,
workingDirectory: reviewCoverageDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'review-coverage-audit',
runId,
});
@@ -107,7 +108,7 @@ Output the diagram directly.`,
// At minimum, the agent should have read the source and test files
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
expect(readCalls.length).toBeGreaterThan(0);
}, 180_000);
}, CAPTURE_MS);
});
// --- Plan Eng Review Coverage Audit E2E ---
@@ -153,7 +154,7 @@ Output the diagram directly.`,
workingDirectory: planCoverageDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'plan-eng-coverage-audit',
runId,
});
@@ -182,7 +183,7 @@ Output the diagram directly.`,
// At minimum, the agent should have read the source and test files
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
expect(readCalls.length).toBeGreaterThan(0);
}, 180_000);
}, CAPTURE_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
+6 -5
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId, evalsEnabled,
@@ -75,7 +76,7 @@ IMPORTANT:
workingDirectory: csoDir,
maxTurns: 30,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob', 'Agent'],
timeout: 300_000,
timeout: CAPTURE_MS,
});
logCost('cso', result);
@@ -106,7 +107,7 @@ IMPORTANT:
}
recordE2E(evalCollector, 'cso-full-audit', 'e2e-cso', result);
}, 300_000);
}, CAPTURE_MS);
});
describeIfSelected('CSO v2 — diff mode', ['cso-diff-mode'], () => {
@@ -181,7 +182,7 @@ IMPORTANT:
).toBe(true);
recordE2E(evalCollector, 'cso-diff-mode', 'e2e-cso', result);
}, 400_000);
}, CAPTURE_LONG_MS);
});
describeIfSelected('CSO v2 — infra scope', ['cso-infra-scope'], () => {
@@ -245,7 +246,7 @@ IMPORTANT:
workingDirectory: csoInfraDir,
maxTurns: 30,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
});
logCost('cso', result);
@@ -259,5 +260,5 @@ IMPORTANT:
).toBe(true);
recordE2E(evalCollector, 'cso-infra-scope', 'e2e-cso', result);
}, 360_000);
}, CAPTURE_LONG_MS);
});
+13 -12
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, browseBin, runId, evalsEnabled,
@@ -67,7 +68,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`,
workingDirectory: landDir,
maxTurns: 20,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'land-and-deploy-workflow',
runId,
});
@@ -85,7 +86,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`,
const reportDir = path.join(landDir, '.gstack', 'deploy-reports');
expect(fs.existsSync(reportDir)).toBe(true);
}, 180_000);
}, CAPTURE_MS);
});
// --- Land-and-Deploy First-Run E2E ---
@@ -148,7 +149,7 @@ Just demonstrate the first-run dry-run output.`,
workingDirectory: firstRunDir,
maxTurns: 20,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'land-and-deploy-first-run',
runId,
});
@@ -167,7 +168,7 @@ Just demonstrate the first-run dry-run output.`,
const reportContent = fs.readFileSync(path.join(reportDir, reportFiles[0]), 'utf-8');
const hasPlatform = reportContent.toLowerCase().includes('fly') || reportContent.toLowerCase().includes('first-run-app');
expect(hasPlatform).toBe(true);
}, 180_000);
}, CAPTURE_MS);
});
// --- Land-and-Deploy Review Gate E2E ---
@@ -226,7 +227,7 @@ Show what the readiness gate output would look like.`,
workingDirectory: reviewDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'land-and-deploy-review-gate',
runId,
});
@@ -246,7 +247,7 @@ Show what the readiness gate output would look like.`,
const hasReviewMention = reportContent.toLowerCase().includes('review') ||
reportContent.toLowerCase().includes('not run');
expect(hasReviewMention).toBe(true);
}, 180_000);
}, CAPTURE_MS);
});
// --- Canary skill E2E ---
@@ -294,7 +295,7 @@ Just create the directory structure and report files showing the correct schema.
workingDirectory: canaryDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'canary-workflow',
runId,
});
@@ -307,7 +308,7 @@ Just create the directory structure and report files showing the correct schema.
const reportDir = path.join(canaryDir, '.gstack', 'canary-reports');
const files = fs.readdirSync(reportDir, { recursive: true }) as string[];
expect(files.length).toBeGreaterThan(0);
}, 180_000);
}, CAPTURE_MS);
});
// --- Benchmark skill E2E ---
@@ -357,7 +358,7 @@ Just create the files showing the correct schema and report format.`,
workingDirectory: benchDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'benchmark-workflow',
runId,
});
@@ -372,7 +373,7 @@ Just create the files showing the correct schema and report format.`,
const files = fs.readdirSync(baselineDir);
expect(files.length).toBeGreaterThan(0);
}
}, 180_000);
}, CAPTURE_MS);
});
// --- Setup-Deploy skill E2E ---
@@ -418,7 +419,7 @@ Just detect the platform and write the config.`,
workingDirectory: setupDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'setup-deploy-workflow',
runId,
});
@@ -434,7 +435,7 @@ Just detect the platform and write the config.`,
expect(content.toLowerCase()).toContain('fly');
expect(content).toContain('my-cool-app');
expect(content).toContain('Deploy Configuration');
}, 180_000);
}, CAPTURE_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
+15 -14
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import { callJudge } from './helpers/llm-judge';
import {
@@ -113,7 +114,7 @@ Skip research — work from your design knowledge. Skip the font preview page. S
Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
workingDirectory: designDir,
maxTurns: 20,
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'design-consultation-core',
runId,
model: 'claude-opus-4-7',
@@ -178,7 +179,7 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
const claude = fs.readFileSync(claudePath, 'utf-8');
expect(claude.toLowerCase()).toContain('design.md');
}
}, 420_000);
}, CAPTURE_LONG_MS);
testConcurrentIfSelected('design-consultation-research', async () => {
// Test WebSearch integration — research phase only, no DESIGN.md generation
@@ -202,7 +203,7 @@ Do NOT generate a full DESIGN.md — just research notes.`,
// queued past the budget under concurrent API load. 90s budgets cannot
// absorb one slow first completion; 300s is the repo's standard floor
// for CI SDK tests. Outer timeout below rises to 360s for headroom.
timeout: 300_000,
timeout: CAPTURE_MS,
testName: 'design-consultation-research',
runId,
});
@@ -232,7 +233,7 @@ Do NOT generate a full DESIGN.md — just research notes.`,
}
try { fs.rmSync(researchDir, { recursive: true, force: true }); } catch {}
}, 360_000);
}, CAPTURE_LONG_MS);
testConcurrentIfSelected('design-consultation-existing', async () => {
// Pre-create a minimal DESIGN.md (independent of core test)
@@ -250,7 +251,7 @@ There is already a DESIGN.md in this repo. Update it with a complete design syst
Skip research. Skip font preview. Skip any AskUserQuestion calls this is non-interactive.`,
workingDirectory: designDir,
maxTurns: 20,
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'design-consultation-existing',
runId,
model: 'claude-opus-4-7',
@@ -279,7 +280,7 @@ Skip research. Skip font preview. Skip any AskUserQuestion calls — this is non
expect(hasColor).toBe(true);
expect(hasSpacing).toBe(true);
}
}, 420_000);
}, CAPTURE_LONG_MS);
testConcurrentIfSelected('design-consultation-preview', async () => {
// Test preview HTML generation only — no DESIGN.md (covered by core test)
@@ -302,7 +303,7 @@ Do NOT write DESIGN.md — only the preview HTML.`,
maxTurns: 8,
// 300s, not 90s: this is the test that failed 3x at 0 turns/$0.00/93s
// on PR #2533 CI — see the research test's comment for the class.
timeout: 300_000,
timeout: CAPTURE_MS,
testName: 'design-consultation-preview',
runId,
});
@@ -331,7 +332,7 @@ Do NOT write DESIGN.md — only the preview HTML.`,
}
try { fs.rmSync(previewDir, { recursive: true, force: true }); } catch {}
}, 360_000);
}, CAPTURE_LONG_MS);
});
// --- Plan Design Review E2E (plan-mode) ---
@@ -398,7 +399,7 @@ Skip the preamble bash block. Skip any AskUserQuestion calls — this is non-int
IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan review, not a live site audit. Just read the plan file, review it, and edit it to fix the gaps.`,
workingDirectory: reviewDir,
maxTurns: 15,
timeout: 300_000,
timeout: CAPTURE_MS,
testName: 'plan-design-review-plan-mode',
runId,
});
@@ -437,7 +438,7 @@ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan
} finally {
try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {}
}
}, 360_000);
}, CAPTURE_LONG_MS);
testConcurrentIfSelected('plan-design-review-no-ui-scope', async () => {
const reviewDir = setupReviewDir();
@@ -472,7 +473,7 @@ Skip the preamble bash block. Skip any AskUserQuestion calls — this is non-int
IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan review, not a live site audit.`,
workingDirectory: reviewDir,
maxTurns: 10,
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'plan-design-review-no-ui-scope',
runId,
});
@@ -496,7 +497,7 @@ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan
} finally {
try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {}
}
}, 240_000);
}, CAPTURE_MS);
});
// --- Design Review E2E (live-site audit + fix) ---
@@ -602,7 +603,7 @@ Read design-review/SKILL.md for the design review + fix workflow.
Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion calls this is non-interactive. Fix up to 3 issues max. Write your report to ./design-audit.md.`,
workingDirectory: qaDesignDir,
maxTurns: 30,
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'design-review-fix',
runId,
});
@@ -634,7 +635,7 @@ Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion call
console.warn('No design-audit.md generated');
}
console.log(`Design fix commits: ${designFixCommits.length}`);
}, 420_000);
}, CAPTURE_LONG_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
+5 -4
View File
@@ -17,6 +17,7 @@
* with its preamble.
*/
import { describe, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
@@ -73,7 +74,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring-
workingDirectory: dir,
maxTurns: 25,
allowedTools: ['Bash', 'Read', 'Write'],
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'diagram-triplet',
runId,
});
@@ -98,7 +99,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring-
} finally {
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
}
}, 300_000);
}, CAPTURE_MS);
testConcurrentIfSelected('diagram-authoring-quality', async () => {
const dir = setupDir('diagram-quality-');
@@ -111,7 +112,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring-
workingDirectory: dir,
maxTurns: 25,
allowedTools: ['Bash', 'Read', 'Write'],
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'diagram-authoring-quality',
runId,
});
@@ -149,5 +150,5 @@ Respond with JSON: {"score": N, "reasoning": "..."}`,
} finally {
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
}
}, 300_000);
}, CAPTURE_MS);
});
+3 -2
View File
@@ -14,6 +14,7 @@
*/
import { expect, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
@@ -55,7 +56,7 @@ async function detectVia(workDir: string, testName: string): Promise<string> {
workingDirectory: workDir,
maxTurns: 3,
allowedTools: ['Bash'],
timeout: 120_000,
timeout: JUDGE_MS,
testName,
runId,
model: MODEL,
@@ -91,7 +92,7 @@ describeIfSelected('first-run scaffold detection (E2E)', ['first-task-scaffold']
fs.rmSync(nodeDir, { recursive: true, force: true });
fs.rmSync(greenDir, { recursive: true, force: true });
}
}, 300_000);
}, CAPTURE_MS);
});
afterAll(() => finalizeEvalCollector(evalCollector));
@@ -29,6 +29,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS } from './helpers/eval-budgets';
import { execFileSync } from 'child_process';
import { mkdtempSync, rmSync } from 'fs';
import { tmpdir } from 'os';
@@ -156,7 +157,7 @@ ${body}`;
expect(retrieved).not.toContain('page_not_found');
expect(retrieved).not.toContain('Page not found');
},
120_000,
JUDGE_MS,
);
},
);
+5 -4
View File
@@ -31,6 +31,7 @@
*/
import { expect, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
@@ -94,7 +95,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
workingDirectory: workDir,
maxTurns: 3,
allowedTools: ['Bash'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'hermetic-canary',
runId,
model: CANARY_MODEL,
@@ -129,7 +130,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
}
fs.rmSync(workDir, { recursive: true, force: true });
}
}, 180_000);
}, CAPTURE_MS);
testIfSelected('hermetic-sentinel', async () => {
if (!process.env.ANTHROPIC_API_KEY) {
@@ -158,7 +159,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
workingDirectory: workDir,
maxTurns: 3,
allowedTools: ['Bash'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'hermetic-sentinel',
runId,
model: CANARY_MODEL,
@@ -188,7 +189,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
fs.rmSync(workDir, { recursive: true, force: true });
fs.rmSync(poisonRoot, { recursive: true, force: true });
}
}, 180_000);
}, CAPTURE_MS);
});
afterAll(() => finalizeEvalCollector(evalCollector));
+2 -1
View File
@@ -21,6 +21,7 @@
// intentionally machine-specific.
import { describe, test, expect } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { spawnSync } from 'child_process';
import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync, unlinkSync } from 'fs';
import { tmpdir } from 'os';
@@ -856,7 +857,7 @@ describe('ios device deployment (explicit opt-in)', () => {
keepalive?.stop();
rmSync(workDir, { recursive: true, force: true });
}
}, 600_000);
}, CAPTURE_LONG_MS);
});
// Always-on instructions if not paired. Surfaces actionable steps even when
+4 -3
View File
@@ -18,6 +18,7 @@
// gated (no compilation step for DebugBridgeCore/UI)
import { describe, test, expect } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { spawnSync } from 'child_process';
import { readFileSync } from 'fs';
import { join } from 'path';
@@ -321,7 +322,7 @@ describeIfSwift('swift build invariants', () => {
console.error('swift build stderr:', r.stderr?.toString().slice(0, 4000));
}
expect(r.status).toBe(0);
}, 180_000);
}, CAPTURE_MS);
test('XCTest suite for StateServer passes (validates real Swift impl)', () => {
const r = spawnSync('swift', ['test', '--filter', 'DebugBridgeCoreTests'], {
@@ -342,7 +343,7 @@ describeIfSwift('swift build invariants', () => {
// Guard against an empty pass-by-no-tests (filter typo / target rename):
// we expect at least one StateServer smoke test to actually execute.
expect(combined).toContain('StateServerSmokeTests');
}, 240_000);
}, CAPTURE_MS);
// Codex-flagged: Release-build guard must be STRUCTURAL, not advisory.
// The Package.swift's `.when(configuration: .debug)` setting causes Swift
@@ -386,5 +387,5 @@ describeIfSwift('swift build invariants', () => {
}
}
expect(foundForbidden).toBe(0);
}, 300_000);
}, CAPTURE_MS);
});
+3 -2
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId, evalsEnabled,
@@ -103,7 +104,7 @@ IMPORTANT:
workingDirectory: workDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'learnings-show',
runId,
});
@@ -134,5 +135,5 @@ IMPORTANT:
} else {
console.warn(`Only ${foundCount}/3 learnings found (N+1: ${mentionsNPlusOne}, cache: ${mentionsCache}, rubocop: ${mentionsRubocop})`);
}
}, 180_000);
}, CAPTURE_MS);
});
@@ -17,6 +17,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillObservation, planFileHasDecisionsSection } from './helpers/claude-pty-runner';
@@ -30,7 +31,7 @@ describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => {
skillName: 'office-hours',
inPlanMode: true,
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (
@@ -55,5 +56,5 @@ describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => {
}
}
expect(['asked', 'plan_ready']).toContain(obs.outcome);
}, 360_000);
}, CAPTURE_LONG_MS);
});
@@ -36,6 +36,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { execFileSync, spawnSync } from 'child_process';
import {
chmodSync,
@@ -122,6 +123,8 @@ describeIfSelected(
'--respect-detection',
],
{
// LIVE-REPO CWD: gen-skill-docs regenerates the in-repo
// office-hours SKILL.md + section (snapshotted/restored in finally).
cwd: ROOT,
env: { ...process.env, GSTACK_HOME: tmpHome },
stdio: ['ignore', 'pipe', 'pipe'],
@@ -220,7 +223,7 @@ Generate the design doc per Phase 5. The feature-slug value to substitute into t
This is a test of the brain-writeback path. Do NOT skip the gbrain save step under any circumstance the runtime guard ("skip if gbrain not on PATH") does NOT apply here because gbrain IS available. Do NOT explore gbrain --help; follow the SAVE_RESULTS template's exact CLI shape. If you encounter any AskUserQuestion, auto-decide recommended.`,
workingDirectory: workDir,
maxTurns: 12,
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'office-hours-brain-writeback',
runId,
model: 'claude-sonnet-4-6',
@@ -313,7 +316,7 @@ This is a test of the brain-writeback path. Do NOT skip the gbrain save step und
);
}
},
420_000,
CAPTURE_LONG_MS,
);
},
);
+3 -2
View File
@@ -20,6 +20,7 @@
* test turns out stable.
*/
import { expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId,
@@ -133,7 +134,7 @@ ${captureInstruction(outFile)}
After writing the file with that ONE Phase 4 question, stop. Do not continue to Phase 4.5 or Phase 5.`,
workingDirectory: workDir,
maxTurns: 12,
timeout: 300_000,
timeout: CAPTURE_MS,
testName: 'office-hours-phase4-fork',
runId,
model: 'claude-opus-4-7',
@@ -162,7 +163,7 @@ After writing the file with that ONE Phase 4 question, stop. Do not continue to
result,
passed: ['success', 'error_max_turns'].includes(result.exitReason),
});
}, 360_000);
}, CAPTURE_LONG_MS);
});
afterAll(async () => {
+5 -4
View File
@@ -10,6 +10,7 @@
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, browseBin, runId, evalsEnabled,
@@ -71,7 +72,7 @@ Assume the founder has already answered Q1 (strongest evidence = "got on a waitl
Write Q3 output the forcing question you would ask this founder to ${workDir}/q3.md. Write ONLY the question prose. No conversational wrapper, no meta-commentary, no Q1/Q2 recap.`,
workingDirectory: workDir,
maxTurns: 8,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'office-hours-forcing-energy',
runId,
model: 'claude-sonnet-4-6',
@@ -94,7 +95,7 @@ Write Q3 output — the forcing question you would ask this founder — to ${wor
console.log('Forcing energy scores:', JSON.stringify(scores, null, 2));
expect(scores.axis_a).toBeGreaterThanOrEqual(4); // stacking_preserved
expect(scores.axis_b).toBeGreaterThanOrEqual(4); // domain_matched_consequence
}, 360_000);
}, CAPTURE_LONG_MS);
});
// --- Office Hours builder-mode wildness ---
@@ -143,7 +144,7 @@ The user has confirmed the basic idea is "TypeScript + D3 web tool, start with J
Write your response the three adjacent unlocks to ${workDir}/unlocks.md. Write ONLY the response prose. No meta-commentary, no mode recap. Lead with the fun; let me edit it down later.`,
workingDirectory: workDir,
maxTurns: 8,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'office-hours-builder-wildness',
runId,
model: 'claude-sonnet-4-6',
@@ -166,7 +167,7 @@ Write your response — the three adjacent unlocks — to ${workDir}/unlocks.md.
console.log('Builder wildness scores:', JSON.stringify(scores, null, 2));
expect(scores.axis_a).toBeGreaterThanOrEqual(4); // unexpected_combinations
expect(scores.axis_b).toBeGreaterThanOrEqual(4); // excitement_over_optimization
}, 360_000);
}, CAPTURE_LONG_MS);
});
// Finalize eval collector for this file
+10 -5
View File
@@ -18,6 +18,7 @@
*/
import { describe, test, expect, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import { EvalCollector } from './helpers/eval-store';
import { extractSkillHead } from './helpers/skill-fixture';
@@ -67,6 +68,8 @@ function mkEvalRoot(suffix: string, includeOverlay: boolean): string {
const result = spawnSync(
'bun',
['run', 'scripts/gen-skill-docs.ts', '--model', includeOverlay ? 'opus-4-7' : 'claude'],
// LIVE-REPO CWD: gen-skill-docs reads .tmpl sources and regenerates the
// in-repo SKILL.md files (restored to default in afterAll below).
{ cwd: ROOT, stdio: 'pipe', encoding: 'utf-8', timeout: 60_000 },
);
if (result.status !== 0) {
@@ -169,6 +172,8 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
// whichever model ran last. Reset to the default (claude) so the tree
// matches what would be checked in.
spawnSync('bun', ['run', 'scripts/gen-skill-docs.ts'], {
// LIVE-REPO CWD: restores the in-repo SKILL.md files to the default
// model render after mkEvalRoot's --model regens.
cwd: ROOT,
stdio: 'pipe',
timeout: 60_000,
@@ -200,7 +205,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
workingDirectory: armA,
maxTurns: 5,
allowedTools: ['Read', 'Bash', 'Glob', 'Grep'],
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'fanout-arm-overlay-on',
runId,
model: OPUS_47,
@@ -210,7 +215,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
workingDirectory: armB,
maxTurns: 5,
allowedTools: ['Read', 'Bash', 'Glob', 'Grep'],
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'fanout-arm-overlay-off',
runId,
model: OPUS_47,
@@ -258,7 +263,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
fs.rmSync(armB, { recursive: true, force: true });
}
},
240_000,
CAPTURE_MS,
);
test(
@@ -277,7 +282,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
workingDirectory: root,
maxTurns: 3,
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
timeout: 90_000,
timeout: JUDGE_MS,
testName: `routing-${c.name}`,
runId,
model: OPUS_47,
@@ -344,6 +349,6 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
fs.rmSync(root, { recursive: true, force: true });
}
},
360_000,
CAPTURE_LONG_MS,
);
});
+49 -48
View File
@@ -18,6 +18,8 @@
import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import {
runPlanSkillCounting,
ceoStep0Boundary,
@@ -62,8 +64,8 @@ const N_PAIRED = 2;
const FLOOR_PAIRED = 2;
const CEILING_PAIRED = 4;
const PLAN_CEO_5_FINDINGS = [
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-ceo.md (use Edit/Write to that exact path).',
const planCeo5Findings = (planPath: string) => [
`Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
'',
'# Plan: Payment Processing Integration',
'',
@@ -88,8 +90,8 @@ const PLAN_CEO_5_FINDINGS = [
'order in a loop.',
].join('\n');
const PLAN_CEO_2_PAIRED_FINDINGS = [
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-ceo-paired.md (use Edit/Write to that exact path).',
const planCeo2PairedFindings = (planPath: string) => [
`Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
'',
'# Plan: Payment Processing — Test Coverage',
'',
@@ -102,32 +104,31 @@ const PLAN_CEO_2_PAIRED_FINDINGS = [
'the success path is correctness, the failure path is graceful degradation.',
].join('\n');
const PLAN_CEO_PATH = '/tmp/gstack-test-plan-ceo.md';
const PLAN_CEO_PAIRED_PATH = '/tmp/gstack-test-plan-ceo-paired.md';
describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () => {
test(
`5-finding plan emits ${FLOOR_DISTINCT}-${CEILING_DISTINCT} review-phase AskUserQuestions`,
async () => {
try {
fs.rmSync(PLAN_CEO_PATH, { force: true });
} catch {
/* best-effort */
}
const obs = await runPlanSkillCounting({
skillName: 'plan-ceo-review',
slashCommand: '/plan-ceo-review',
followUpPrompt: PLAN_CEO_5_FINDINGS,
isLastStep0AUQ: ceoStep0Boundary,
reviewCountCeiling: CEILING_DISTINCT + 1, // hard cap above assertion ceiling
firstAUQPick: pickSkipInterview, // bypass scope-selection, route to review
cwd: process.cwd(),
timeoutMs: 1_500_000, // 25 min
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
// Per-run artifact dir: a hardcoded shared /tmp path collides under
// --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally-
// rmSync deletes this run's artifact → spurious D19 failure).
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-'));
const planPath = path.join(tmpDir, 'gstack-test-plan-ceo.md');
try {
const obs = await runPlanSkillCounting({
skillName: 'plan-ceo-review',
slashCommand: '/plan-ceo-review',
followUpPrompt: planCeo5Findings(planPath),
isLastStep0AUQ: ceoStep0Boundary,
reviewCountCeiling: CEILING_DISTINCT + 1, // hard cap above assertion ceiling
firstAUQPick: pickSkipInterview, // bypass scope-selection, route to review
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000, // 25 min
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
throw new Error(
`plan-ceo-review finding-count FAILED: outcome=${obs.outcome}\n` +
@@ -166,19 +167,19 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
}
// D19: review report at bottom of plan file.
if (!fs.existsSync(PLAN_CEO_PATH)) {
if (!fs.existsSync(planPath)) {
throw new Error(
`D19 FAIL: agent did not produce expected plan file at ${PLAN_CEO_PATH}.\n` +
`D19 FAIL: agent did not produce expected plan file at ${planPath}.\n` +
`Either the agent ignored the path instruction in the follow-up prompt, or\n` +
`the helper exited before the agent wrote the file. ` +
`outcome=${obs.outcome} review=${obs.reviewCount}`,
);
}
const planContent = fs.readFileSync(PLAN_CEO_PATH, 'utf-8');
const planContent = fs.readFileSync(planPath, 'utf-8');
const verdict = assertReviewReportAtBottom(planContent);
if (!verdict.ok) {
throw new Error(
`D19 FAIL: plan file at ${PLAN_CEO_PATH} ${verdict.reason}\n` +
`D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` +
(verdict.trailingHeadings
? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n`
: '') +
@@ -187,36 +188,36 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
}
} finally {
try {
fs.rmSync(PLAN_CEO_PATH, { force: true });
fs.rmSync(tmpDir, { recursive: true, force: true });
} catch {
/* best-effort */
}
}
},
1_700_000,
1_500_000 /* physical ceiling: the 25-min CI job + 1800s shard wall cap what can actually execute */,
);
test(
`paired-finding positive control: ${N_PAIRED} related findings produce ${FLOOR_PAIRED}-${CEILING_PAIRED} AskUserQuestions`,
async () => {
try {
fs.rmSync(PLAN_CEO_PAIRED_PATH, { force: true });
} catch {
/* best-effort */
}
const obs = await runPlanSkillCounting({
skillName: 'plan-ceo-review',
slashCommand: '/plan-ceo-review',
followUpPrompt: PLAN_CEO_2_PAIRED_FINDINGS,
isLastStep0AUQ: ceoStep0Boundary,
reviewCountCeiling: CEILING_PAIRED + 1,
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
// Per-run artifact dir — see the distinct-findings test above.
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-paired-'));
const planPath = path.join(tmpDir, 'gstack-test-plan-ceo-paired.md');
try {
const obs = await runPlanSkillCounting({
skillName: 'plan-ceo-review',
slashCommand: '/plan-ceo-review',
followUpPrompt: planCeo2PairedFindings(planPath),
isLastStep0AUQ: ceoStep0Boundary,
reviewCountCeiling: CEILING_PAIRED + 1,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
throw new Error(
`paired-finding control FAILED: outcome=${obs.outcome}\n` +
@@ -242,12 +243,12 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
}
} finally {
try {
fs.rmSync(PLAN_CEO_PAIRED_PATH, { force: true });
fs.rmSync(tmpDir, { recursive: true, force: true });
} catch {
/* best-effort */
}
}
},
1_700_000,
1_500_000 /* physical ceiling: the 25-min CI job + 1800s shard wall cap what can actually execute */,
);
});
@@ -5,6 +5,7 @@
*/
import { test } from 'bun:test';
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
import { FORCING_FLOOR_CEO } from './fixtures/forcing-finding-seeds';
@@ -19,8 +20,10 @@ describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => {
skillName: 'plan-ceo-review',
slashCommand: '/plan-ceo-review',
followUpPrompt: FORCING_FLOOR_CEO,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 600_000,
timeoutMs: CAPTURE_LONG_MS,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
@@ -32,6 +35,6 @@ describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => {
);
}
},
660_000,
PTY_MS,
);
});
+3 -2
View File
@@ -31,6 +31,7 @@
*/
import { test } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import {
launchClaudePty,
@@ -151,7 +152,7 @@ describeE2E('/plan-ceo-review mode routing (gate)', () => {
async () => {
const session = await launchClaudePty({
permissionMode: 'plan',
timeoutMs: 540_000,
timeoutMs: CAPTURE_LONG_MS,
seedSkills: true,
});
try {
@@ -207,7 +208,7 @@ describeE2E('/plan-ceo-review mode routing (gate)', () => {
await session.close();
}
},
600_000,
CAPTURE_LONG_MS,
);
}
});
+2 -1
View File
@@ -34,6 +34,7 @@
*/
import { test } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import {
runPlanSkillObservation,
@@ -77,5 +78,5 @@ describeE2E('plan-ceo-review plan-mode smoke (gate)', () => {
);
}
assertReportAtBottomIfPlanWritten(obs);
}, 480_000);
}, CAPTURE_LONG_MS);
});
@@ -25,6 +25,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import {
setupSkillDir,
@@ -87,6 +88,6 @@ describeE2E('/plan-ceo-review section-loading E2E (periodic, SDK capture)', () =
// Guard against an empty pass: the report must have real content.
expect(output.trim().length).toBeGreaterThan(200);
},
360_000,
CAPTURE_LONG_MS,
);
});
+28 -18
View File
@@ -35,6 +35,8 @@
import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import {
runPlanSkillCounting,
ceoStep0Boundary,
@@ -46,30 +48,38 @@ const describeE2E = describeE2ETier('periodic');
const N = 5;
const FLOOR = N - 1; // 4 — must fire at least one AUQ per non-dropped option
const PLAN_PATH = '/tmp/gstack-test-plan-ceo-split-overflow.md';
/** Plan-file target baked into the FORCING_SPLIT_OVERFLOW_CEO fixture prompt.
* Rewritten per-run to a mkdtemp path so concurrent runs (--retry,
* EVALS_JOBS>1, sibling worktrees) never share one /tmp artifact. */
const FIXTURE_PLAN_PATH = '/tmp/gstack-test-plan-ceo-split-overflow.md';
describeE2E('/plan-ceo-review split-overflow regression (periodic)', () => {
test(
`5-option scope decision emits >= ${FLOOR} review-phase AskUserQuestions (no dropping)`,
async () => {
try {
fs.rmSync(PLAN_PATH, { force: true });
} catch {
/* best-effort */
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-split-overflow-'));
const planPath = path.join(tmpDir, 'gstack-test-plan-ceo-split-overflow.md');
const followUpPrompt = FORCING_SPLIT_OVERFLOW_CEO.replaceAll(FIXTURE_PLAN_PATH, planPath);
if (!followUpPrompt.includes(planPath)) {
throw new Error(
`fixture drift: FORCING_SPLIT_OVERFLOW_CEO no longer contains ${FIXTURE_PLAN_PATH} — update FIXTURE_PLAN_PATH`,
);
}
const obs = await runPlanSkillCounting({
skillName: 'plan-ceo-review',
slashCommand: '/plan-ceo-review',
followUpPrompt: FORCING_SPLIT_OVERFLOW_CEO,
isLastStep0AUQ: ceoStep0Boundary,
reviewCountCeiling: N + 3, // hard cap above floor + tolerance
cwd: process.cwd(),
timeoutMs: 1_500_000, // 25 min
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
try {
const obs = await runPlanSkillCounting({
skillName: 'plan-ceo-review',
slashCommand: '/plan-ceo-review',
followUpPrompt,
isLastStep0AUQ: ceoStep0Boundary,
reviewCountCeiling: N + 3, // hard cap above floor + tolerance
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000, // 25 min
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
throw new Error(
`split-overflow test FAILED: outcome=${obs.outcome}\n` +
@@ -97,12 +107,12 @@ describeE2E('/plan-ceo-review split-overflow regression (periodic)', () => {
}
} finally {
try {
fs.rmSync(PLAN_PATH, { force: true });
fs.rmSync(tmpDir, { recursive: true, force: true });
} catch {
/* best-effort */
}
}
},
1_700_000,
1_500_000 /* physical ceiling: the 25-min CI job + 1800s shard wall cap what can actually execute */,
);
});
@@ -11,6 +11,8 @@
import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import {
runPlanSkillCounting,
designStep0Boundary,
@@ -23,8 +25,8 @@ const N = 5;
const FLOOR = N - 1;
const CEILING = N + 2;
const PLAN_DESIGN_5_FINDINGS = [
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-design.md (use Edit/Write to that exact path).',
const planDesign5Findings = (planPath: string) => [
`Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
'',
'# Plan: Settings Page UI redesign',
'',
@@ -50,30 +52,30 @@ const PLAN_DESIGN_5_FINDINGS = [
'see a frozen page; we should add a spinner or skeleton state.',
].join('\n');
const PLAN_DESIGN_PATH = '/tmp/gstack-test-plan-design.md';
describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)', () => {
test(
`5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`,
async () => {
try {
fs.rmSync(PLAN_DESIGN_PATH, { force: true });
} catch {
/* best-effort */
}
const obs = await runPlanSkillCounting({
skillName: 'plan-design-review',
slashCommand: '/plan-design-review',
followUpPrompt: PLAN_DESIGN_5_FINDINGS,
isLastStep0AUQ: designStep0Boundary,
reviewCountCeiling: CEILING + 1,
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
// Per-run artifact dir: a hardcoded shared /tmp path collides under
// --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally-
// rmSync deletes this run's artifact → spurious D19 failure).
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-design-'));
const planPath = path.join(tmpDir, 'gstack-test-plan-design.md');
try {
const obs = await runPlanSkillCounting({
skillName: 'plan-design-review',
slashCommand: '/plan-design-review',
followUpPrompt: planDesign5Findings(planPath),
isLastStep0AUQ: designStep0Boundary,
reviewCountCeiling: CEILING + 1,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
throw new Error(
`plan-design-review finding-count FAILED: outcome=${obs.outcome}\n` +
@@ -105,17 +107,17 @@ describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)',
);
}
if (!fs.existsSync(PLAN_DESIGN_PATH)) {
if (!fs.existsSync(planPath)) {
throw new Error(
`D19 FAIL: agent did not produce expected plan file at ${PLAN_DESIGN_PATH}. ` +
`D19 FAIL: agent did not produce expected plan file at ${planPath}. ` +
`outcome=${obs.outcome} review=${obs.reviewCount}`,
);
}
const planContent = fs.readFileSync(PLAN_DESIGN_PATH, 'utf-8');
const planContent = fs.readFileSync(planPath, 'utf-8');
const verdict = assertReviewReportAtBottom(planContent);
if (!verdict.ok) {
throw new Error(
`D19 FAIL: plan file at ${PLAN_DESIGN_PATH} ${verdict.reason}\n` +
`D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` +
(verdict.trailingHeadings
? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n`
: '') +
@@ -124,12 +126,12 @@ describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)',
}
} finally {
try {
fs.rmSync(PLAN_DESIGN_PATH, { force: true });
fs.rmSync(tmpDir, { recursive: true, force: true });
} catch {
/* best-effort */
}
}
},
1_700_000,
1_500_000 /* physical ceiling: the 25-min CI job + 1800s shard wall cap what can actually execute */,
);
});
@@ -5,6 +5,7 @@
*/
import { test } from 'bun:test';
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
import { FORCING_FLOOR_DESIGN } from './fixtures/forcing-finding-seeds';
@@ -19,8 +20,10 @@ describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => {
skillName: 'plan-design-review',
slashCommand: '/plan-design-review',
followUpPrompt: FORCING_FLOOR_DESIGN,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 600_000,
timeoutMs: CAPTURE_LONG_MS,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
@@ -32,6 +35,6 @@ describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => {
);
}
},
660_000,
PTY_MS,
);
});
+5 -4
View File
@@ -10,6 +10,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import {
runPlanSkillObservation,
@@ -46,7 +47,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
const obs = await runPlanSkillObservation({
skillName: 'plan-design-review',
inPlanMode: true,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
@@ -59,7 +60,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
}
expect(['asked', 'plan_ready']).toContain(obs.outcome);
assertReportAtBottomIfPlanWritten(obs);
}, 360_000);
}, CAPTURE_LONG_MS);
// Plan-mode scope-gate bypass: with a seeded UI-heavy plan in plan mode,
// the gate must NOT render its "What should I review?" menu — it
@@ -71,7 +72,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
skillName: 'plan-design-review',
inPlanMode: true,
initialPlanContent: SEED_PLAN_UI_HEAVY,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (
@@ -95,5 +96,5 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
// though the seed arrives as a pasted user message).
expect(obs.scopeGateQuestionObserved ?? false).toBe(false);
expect(obs.scopeGateAutoSelectObserved ?? false).toBe(true);
}, 360_000);
}, CAPTURE_LONG_MS);
});
+5 -2
View File
@@ -20,6 +20,7 @@
*/
import { test } from 'bun:test';
import { PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as path from 'path';
import {
@@ -43,8 +44,10 @@ describeE2E('/plan-design-review with UI scope (gate)', () => {
const session = await launchClaudePty({
permissionMode: 'plan',
// LIVE-REPO CWD: PTY session needs the repo cwd — skill registry,
// hermetic pre-trusted dir, and the repo-relative fixture path above.
cwd: ROOT,
timeoutMs: 720_000,
timeoutMs: PTY_MS,
seedSkills: true,
});
@@ -150,6 +153,6 @@ describeE2E('/plan-design-review with UI scope (gate)', () => {
);
}
},
780_000,
PTY_MS,
);
});
+28 -26
View File
@@ -11,6 +11,8 @@
import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import {
runPlanSkillCounting,
devexStep0Boundary,
@@ -23,8 +25,8 @@ const N = 5;
const FLOOR = N - 1;
const CEILING = N + 2;
const PLAN_DEVEX_5_FINDINGS = [
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-devex.md (use Edit/Write to that exact path).',
const planDevex5Findings = (planPath: string) => [
`Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
'',
'# Plan: Public SDK Beta Launch',
'',
@@ -50,30 +52,30 @@ const PLAN_DEVEX_5_FINDINGS = [
'of solved problems.',
].join('\n');
const PLAN_DEVEX_PATH = '/tmp/gstack-test-plan-devex.md';
describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', () => {
test(
`5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`,
async () => {
try {
fs.rmSync(PLAN_DEVEX_PATH, { force: true });
} catch {
/* best-effort */
}
const obs = await runPlanSkillCounting({
skillName: 'plan-devex-review',
slashCommand: '/plan-devex-review',
followUpPrompt: PLAN_DEVEX_5_FINDINGS,
isLastStep0AUQ: devexStep0Boundary,
reviewCountCeiling: CEILING + 1,
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
// Per-run artifact dir: a hardcoded shared /tmp path collides under
// --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally-
// rmSync deletes this run's artifact → spurious D19 failure).
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-devex-'));
const planPath = path.join(tmpDir, 'gstack-test-plan-devex.md');
try {
const obs = await runPlanSkillCounting({
skillName: 'plan-devex-review',
slashCommand: '/plan-devex-review',
followUpPrompt: planDevex5Findings(planPath),
isLastStep0AUQ: devexStep0Boundary,
reviewCountCeiling: CEILING + 1,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
throw new Error(
`plan-devex-review finding-count FAILED: outcome=${obs.outcome}\n` +
@@ -105,17 +107,17 @@ describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', (
);
}
if (!fs.existsSync(PLAN_DEVEX_PATH)) {
if (!fs.existsSync(planPath)) {
throw new Error(
`D19 FAIL: agent did not produce expected plan file at ${PLAN_DEVEX_PATH}. ` +
`D19 FAIL: agent did not produce expected plan file at ${planPath}. ` +
`outcome=${obs.outcome} review=${obs.reviewCount}`,
);
}
const planContent = fs.readFileSync(PLAN_DEVEX_PATH, 'utf-8');
const planContent = fs.readFileSync(planPath, 'utf-8');
const verdict = assertReviewReportAtBottom(planContent);
if (!verdict.ok) {
throw new Error(
`D19 FAIL: plan file at ${PLAN_DEVEX_PATH} ${verdict.reason}\n` +
`D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` +
(verdict.trailingHeadings
? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n`
: '') +
@@ -124,12 +126,12 @@ describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', (
}
} finally {
try {
fs.rmSync(PLAN_DEVEX_PATH, { force: true });
fs.rmSync(tmpDir, { recursive: true, force: true });
} catch {
/* best-effort */
}
}
},
1_700_000,
1_500_000 /* physical ceiling: the 25-min CI job + 1800s shard wall cap what can actually execute */,
);
});
@@ -5,6 +5,7 @@
*/
import { test } from 'bun:test';
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
import { FORCING_FLOOR_DEVEX } from './fixtures/forcing-finding-seeds';
@@ -19,8 +20,10 @@ describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => {
skillName: 'plan-devex-review',
slashCommand: '/plan-devex-review',
followUpPrompt: FORCING_FLOOR_DEVEX,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 600_000,
timeoutMs: CAPTURE_LONG_MS,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
@@ -32,6 +35,6 @@ describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => {
);
}
},
660_000,
PTY_MS,
);
});
+5 -4
View File
@@ -6,6 +6,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import {
runPlanSkillObservation,
@@ -20,7 +21,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
const obs = await runPlanSkillObservation({
skillName: 'plan-devex-review',
inPlanMode: true,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
@@ -33,7 +34,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
}
expect(['asked', 'plan_ready']).toContain(obs.outcome);
assertReportAtBottomIfPlanWritten(obs);
}, 360_000);
}, CAPTURE_LONG_MS);
// v1.21+ regression: see skill-e2e-plan-ceo-plan-mode.test.ts for the
// contract. Pass envelope is ['asked', 'plan_ready']; failure signals
@@ -44,7 +45,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
skillName: 'plan-devex-review',
inPlanMode: true,
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (
@@ -70,5 +71,5 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
}
expect(['asked', 'plan_ready']).toContain(obs.outcome);
assertReportAtBottomIfPlanWritten(obs);
}, 360_000);
}, CAPTURE_LONG_MS);
});
+28 -26
View File
@@ -11,6 +11,8 @@
import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import {
runPlanSkillCounting,
engStep0Boundary,
@@ -23,8 +25,8 @@ const N = 5;
const FLOOR = N - 1; // 4
const CEILING = N + 2; // 7
const PLAN_ENG_5_FINDINGS = [
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-eng.md (use Edit/Write to that exact path).',
const planEng5Findings = (planPath: string) => [
`Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
'',
'# Plan: Multi-tenant Auth Refactor',
'',
@@ -49,30 +51,30 @@ const PLAN_ENG_5_FINDINGS = [
'SessionMint, AuthCache, RequestPolicy). Worth flagging the complexity check.',
].join('\n');
const PLAN_ENG_PATH = '/tmp/gstack-test-plan-eng.md';
describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', () => {
test(
`5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`,
async () => {
try {
fs.rmSync(PLAN_ENG_PATH, { force: true });
} catch {
/* best-effort */
}
const obs = await runPlanSkillCounting({
skillName: 'plan-eng-review',
slashCommand: '/plan-eng-review',
followUpPrompt: PLAN_ENG_5_FINDINGS,
isLastStep0AUQ: engStep0Boundary,
reviewCountCeiling: CEILING + 1,
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
// Per-run artifact dir: a hardcoded shared /tmp path collides under
// --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally-
// rmSync deletes this run's artifact → spurious D19 failure).
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-eng-'));
const planPath = path.join(tmpDir, 'gstack-test-plan-eng.md');
try {
const obs = await runPlanSkillCounting({
skillName: 'plan-eng-review',
slashCommand: '/plan-eng-review',
followUpPrompt: planEng5Findings(planPath),
isLastStep0AUQ: engStep0Boundary,
reviewCountCeiling: CEILING + 1,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
throw new Error(
`plan-eng-review finding-count FAILED: outcome=${obs.outcome}\n` +
@@ -104,17 +106,17 @@ describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', ()
);
}
if (!fs.existsSync(PLAN_ENG_PATH)) {
if (!fs.existsSync(planPath)) {
throw new Error(
`D19 FAIL: agent did not produce expected plan file at ${PLAN_ENG_PATH}. ` +
`D19 FAIL: agent did not produce expected plan file at ${planPath}. ` +
`outcome=${obs.outcome} review=${obs.reviewCount}`,
);
}
const planContent = fs.readFileSync(PLAN_ENG_PATH, 'utf-8');
const planContent = fs.readFileSync(planPath, 'utf-8');
const verdict = assertReviewReportAtBottom(planContent);
if (!verdict.ok) {
throw new Error(
`D19 FAIL: plan file at ${PLAN_ENG_PATH} ${verdict.reason}\n` +
`D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` +
(verdict.trailingHeadings
? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n`
: '') +
@@ -123,12 +125,12 @@ describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', ()
}
} finally {
try {
fs.rmSync(PLAN_ENG_PATH, { force: true });
fs.rmSync(tmpDir, { recursive: true, force: true });
} catch {
/* best-effort */
}
}
},
1_700_000,
1_500_000 /* physical ceiling: the 25-min CI job + 1800s shard wall cap what can actually execute */,
);
});
@@ -16,6 +16,7 @@
*/
import { test } from 'bun:test';
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
import { FORCING_FLOOR_ENG } from './fixtures/forcing-finding-seeds';
@@ -30,8 +31,10 @@ describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => {
skillName: 'plan-eng-review',
slashCommand: '/plan-eng-review',
followUpPrompt: FORCING_FLOOR_ENG,
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 600_000,
timeoutMs: CAPTURE_LONG_MS,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
@@ -47,6 +50,6 @@ describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => {
);
}
},
660_000,
PTY_MS,
);
});
@@ -27,6 +27,8 @@
import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import {
runPlanSkillCounting,
engStep0Boundary,
@@ -38,30 +40,38 @@ const describeE2E = describeE2ETier('periodic');
const N = 4;
const FLOOR = N - 1; // 3 — agent must fire at least one AUQ per non-batched finding
const PLAN_PATH = '/tmp/gstack-test-plan-eng-batching.md';
/** Plan-file target baked into the FORCING_BATCHING_ENG fixture prompt.
* Rewritten per-run to a mkdtemp path so concurrent runs (--retry,
* EVALS_JOBS>1, sibling worktrees) never share one /tmp artifact. */
const FIXTURE_PLAN_PATH = '/tmp/gstack-test-plan-eng-batching.md';
describeE2E('/plan-eng-review multi-finding batching regression (periodic)', () => {
test(
`4-finding plan emits >= ${FLOOR} review-phase AskUserQuestions (no batching)`,
async () => {
try {
fs.rmSync(PLAN_PATH, { force: true });
} catch {
/* best-effort */
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-eng-batching-'));
const planPath = path.join(tmpDir, 'gstack-test-plan-eng-batching.md');
const followUpPrompt = FORCING_BATCHING_ENG.replaceAll(FIXTURE_PLAN_PATH, planPath);
if (!followUpPrompt.includes(planPath)) {
throw new Error(
`fixture drift: FORCING_BATCHING_ENG no longer contains ${FIXTURE_PLAN_PATH} — update FIXTURE_PLAN_PATH`,
);
}
const obs = await runPlanSkillCounting({
skillName: 'plan-eng-review',
slashCommand: '/plan-eng-review',
followUpPrompt: FORCING_BATCHING_ENG,
isLastStep0AUQ: engStep0Boundary,
reviewCountCeiling: N + 3, // hard cap above floor + tolerance
cwd: process.cwd(),
timeoutMs: 1_500_000, // 25 min
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
try {
const obs = await runPlanSkillCounting({
skillName: 'plan-eng-review',
slashCommand: '/plan-eng-review',
followUpPrompt,
isLastStep0AUQ: engStep0Boundary,
reviewCountCeiling: N + 3, // hard cap above floor + tolerance
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
cwd: process.cwd(),
timeoutMs: 1_500_000, // 25 min
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
throw new Error(
`multi-finding batching test FAILED: outcome=${obs.outcome}\n` +
@@ -85,12 +95,12 @@ describeE2E('/plan-eng-review multi-finding batching regression (periodic)', ()
}
} finally {
try {
fs.rmSync(PLAN_PATH, { force: true });
fs.rmSync(tmpDir, { recursive: true, force: true });
} catch {
/* best-effort */
}
}
},
1_700_000,
1_500_000 /* physical ceiling: the 25-min CI job + 1800s shard wall cap what can actually execute */,
);
});
+5 -4
View File
@@ -6,6 +6,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import {
runPlanSkillObservation,
@@ -50,7 +51,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
const obs = await runPlanSkillObservation({
skillName: 'plan-eng-review',
inPlanMode: true,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
@@ -63,7 +64,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
}
expect(['asked', 'plan_ready']).toContain(obs.outcome);
assertReportAtBottomIfPlanWritten(obs);
}, 360_000);
}, CAPTURE_LONG_MS);
// D3-B / D4-B: when a plan with guaranteed-finding-triggering complexity
// is seeded, the skill MUST fire AskUserQuestion (or fall back to a
@@ -79,7 +80,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
// must use mcp__*__AskUserQuestion (outcome='asked') or fall back to
// writing Decisions ('plan_ready').
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (
@@ -118,5 +119,5 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
// question.
expect(obs.scopeGateQuestionObserved ?? false).toBe(false);
expect(obs.scopeGateAutoSelectObserved ?? false).toBe(true);
}, 360_000);
}, CAPTURE_LONG_MS);
});
+9 -8
View File
@@ -18,6 +18,7 @@
* accordingly.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId,
@@ -134,7 +135,7 @@ ${captureInstruction(outFile)}
After writing the file, stop. Do not continue the review.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-ceo-review-format-mode',
runId,
model: 'claude-opus-4-7',
@@ -160,7 +161,7 @@ After writing the file, stop. Do not continue the review.`,
result,
passed: ['success', 'error_max_turns'].includes(result.exitReason),
});
}, 300_000);
}, CAPTURE_MS);
});
// --- Case 2: plan-ceo-review approach menu (coverage-differentiated) ---
@@ -191,7 +192,7 @@ ${captureInstruction(outFile)}
After writing the file, stop. Do not continue the review.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-ceo-review-format-approach',
runId,
model: 'claude-opus-4-7',
@@ -216,7 +217,7 @@ After writing the file, stop. Do not continue the review.`,
result,
passed: ['success', 'error_max_turns'].includes(result.exitReason),
});
}, 300_000);
}, CAPTURE_MS);
});
// --- Case 3: plan-eng-review coverage-differentiated per-issue AskUserQuestion ---
@@ -250,7 +251,7 @@ ${captureInstruction(outFile)}
After writing the file with that ONE question, stop. Do not continue the review.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-eng-review-format-coverage',
runId,
model: 'claude-opus-4-7',
@@ -275,7 +276,7 @@ After writing the file with that ONE question, stop. Do not continue the review.
result,
passed: ['success', 'error_max_turns'].includes(result.exitReason),
});
}, 300_000);
}, CAPTURE_MS);
});
// --- Case 4: plan-eng-review kind-differentiated per-issue AskUserQuestion ---
@@ -306,7 +307,7 @@ ${captureInstruction(outFile)}
After writing the file with that ONE question, stop. Do not continue the review.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-eng-review-format-kind',
runId,
model: 'claude-opus-4-7',
@@ -332,7 +333,7 @@ After writing the file with that ONE question, stop. Do not continue the review.
result,
passed: ['success', 'error_max_turns'].includes(result.exitReason),
});
}, 300_000);
}, CAPTURE_MS);
});
afterAll(async () => {
+5 -4
View File
@@ -31,6 +31,7 @@
*/
import { test, expect } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
@@ -62,7 +63,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
const obs = await runPlanSkillObservation({
skillName,
inPlanMode: false,
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
// eng/design: force the prose-fallback path. The unconditional
// gate-must-ask assert below pins the render shape the detector
// anchors on, and only the --disallowedTools prose fallback makes
@@ -115,7 +116,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
);
}
}
}, 360_000);
}, CAPTURE_LONG_MS);
}
// Named-target exception (outside plan mode): a pasted draft IS an
@@ -130,7 +131,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
inPlanMode: false,
initialPlanContent: NAMED_TARGET_SEED,
trackTokens: [SEED_TOKEN],
timeoutMs: 300_000,
timeoutMs: CAPTURE_MS,
});
if (
@@ -159,5 +160,5 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
// unreachable outside plan mode (extractPlanFilePath only matches
// plan-mode save renders).
expect(obs.tokensObserved?.[SEED_TOKEN] ?? false).toBe(true);
}, 360_000);
}, CAPTURE_LONG_MS);
});
+9 -8
View File
@@ -27,6 +27,7 @@
* cases will land as follow-up PRs per skill.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId,
@@ -161,7 +162,7 @@ ${captureInstruction(outFile)}
After writing the file, stop.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-review-prosons-format',
runId,
model: 'claude-opus-4-7',
@@ -191,7 +192,7 @@ After writing the file, stop.`,
// (recommended) label on one option
expect(captured).toMatch(RECOMMENDED_LABEL_RE);
}, 300_000);
}, CAPTURE_MS);
});
// --- Case 2: Hard-stop escape NEGATIVE (CT2) ---
@@ -220,7 +221,7 @@ ${captureInstruction(outFile)}
After writing the file, stop.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-review-prosons-hardstop-neg',
runId,
model: 'claude-opus-4-7',
@@ -241,7 +242,7 @@ After writing the file, stop.`,
// Must have real pros and cons (≥2 ✅ + ≥1 ❌ per option)
expect(countChars(captured, '✅')).toBeGreaterThanOrEqual(4);
expect(countChars(captured, '❌')).toBeGreaterThanOrEqual(2);
}, 300_000);
}, CAPTURE_MS);
});
// --- Case 3: Neutral-posture NEGATIVE (CT2) ---
@@ -270,7 +271,7 @@ ${captureInstruction(outFile)}
After writing the file, stop.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-review-prosons-neutral-neg',
runId,
model: 'claude-opus-4-7',
@@ -292,7 +293,7 @@ After writing the file, stop.`,
expect(captured).toMatch(RECOMMENDED_LABEL_RE);
// Recommendation line must contain "because" (concrete reason, not "no preference")
expect(captured).toMatch(/[Rr]ecommendation:.*because/);
}, 300_000);
}, CAPTURE_MS);
});
// --- Case 4: Hard-stop POSITIVE (escape allowed when legitimately one-sided) ---
@@ -321,7 +322,7 @@ ${captureInstruction(outFile)}
After writing the file, stop.`,
workingDirectory: planDir,
maxTurns: 10,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'plan-ceo-review-prosons-cadence',
runId,
model: 'claude-opus-4-7',
@@ -344,7 +345,7 @@ After writing the file, stop.`,
const hasEscape = HARD_STOP_ESCAPE_RE.test(captured);
const hasProsAndCons = countChars(captured, '✅') >= 1 && countChars(captured, '❌') >= 1;
expect(hasEscape || hasProsAndCons).toBe(true);
}, 300_000);
}, CAPTURE_MS);
});
afterAll(async () => {
+3 -2
View File
@@ -1,4 +1,5 @@
import { beforeAll, afterAll, expect } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId,
@@ -151,7 +152,7 @@ IMPORTANT:
workingDirectory: workDir,
maxTurns: 15,
allowedTools: ['Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'plan-tune-inspect',
runId,
});
@@ -184,5 +185,5 @@ IMPORTANT:
if (!noticedOverride) {
console.warn('Agent did not surface override/skip behavior from the log');
}
}, 180_000);
}, CAPTURE_MS);
});
+21 -20
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, browseBin, runId, evalsEnabled,
@@ -103,7 +104,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and
const review = fs.readFileSync(reviewPath, 'utf-8');
expect(review.length).toBeGreaterThan(200);
}
}, 660_000);
}, PTY_MS);
});
// --- Plan CEO Review (SELECTIVE EXPANSION) E2E ---
@@ -171,7 +172,7 @@ Write your complete review directly to ${planDir}/review-output-selective.md
Focus on reviewing the plan content: architecture, error handling, security, and performance.`,
workingDirectory: planDir,
maxTurns: 15,
timeout: 540_000,
timeout: CAPTURE_LONG_MS,
testName: 'plan-ceo-review-selective',
runId,
model: 'claude-opus-4-7',
@@ -188,7 +189,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and
const review = fs.readFileSync(reviewPath, 'utf-8');
expect(review.length).toBeGreaterThan(200);
}
}, 660_000);
}, PTY_MS);
});
// --- Plan CEO Review SCOPE EXPANSION energy (V1.1 mode-posture regression gate) ---
@@ -239,7 +240,7 @@ Choose SCOPE EXPANSION mode. Skip any AskUserQuestion calls — this is non-inte
Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal text no conversational wrapper, no review summary, no mode analysis. Each proposal separated by "---".`,
workingDirectory: planDir,
maxTurns: 15,
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'plan-ceo-review-expansion-energy',
runId,
model: 'claude-opus-4-7',
@@ -270,7 +271,7 @@ Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal
// Pass threshold: 4/5 on both axes (good — matches posture with minor weakness).
expect(scores.axis_a).toBeGreaterThanOrEqual(4); // surface_framing
expect(scores.axis_b).toBeGreaterThanOrEqual(4); // decision_preservation
}, 600_000);
}, CAPTURE_LONG_MS);
});
// --- Plan Eng Review E2E ---
@@ -348,7 +349,7 @@ Write your complete review directly to ${planDir}/review-output.md
Focus on architecture, code quality, tests, and performance sections.`,
workingDirectory: planDir,
maxTurns: 15,
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'plan-eng-review',
runId,
model: 'claude-opus-4-7',
@@ -366,7 +367,7 @@ Focus on architecture, code quality, tests, and performance sections.`,
const review = fs.readFileSync(reviewPath, 'utf-8');
expect(review.length).toBeGreaterThan(200);
}
}, 420_000);
}, CAPTURE_LONG_MS);
});
// --- Plan-Eng-Review Test-Plan Artifact E2E ---
@@ -476,7 +477,7 @@ Write your review to ${planDir}/review-output.md`,
workingDirectory: planDir,
maxTurns: 25,
allowedTools: ['Bash', 'Read', 'Write', 'Glob', 'Grep'],
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'plan-eng-review-artifact',
runId,
model: 'claude-opus-4-7',
@@ -507,7 +508,7 @@ Write your review to ${planDir}/review-output.md`,
if (newFiles.length === 0) {
console.warn('SOFT FAIL: No test-plan artifact written — agent did not follow artifact instructions');
}
}, 420_000);
}, CAPTURE_LONG_MS);
});
// --- Office Hours Spec Review E2E ---
@@ -559,7 +560,7 @@ Write your summary to ${ohDir}/spec-review-summary.md`,
// failures wrote a correct summary on tool-turn 8 and hit the cap on
// the closing text turn (error_max_turns at 9 turns, deterministic).
maxTurns: 12,
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'office-hours-spec-review',
runId,
});
@@ -575,7 +576,7 @@ Write your summary to ${ohDir}/spec-review-summary.md`,
expect(summary).toMatch(/agent|subagent/);
expect(summary).toMatch(/3.*iteration|iteration.*3|maximum.*3/);
}
}, 180_000);
}, CAPTURE_MS);
});
// --- Plan CEO Review Benefits-From E2E ---
@@ -619,7 +620,7 @@ Summarize what happens when no design doc is found — specifically:
Write your summary to ${benefitsDir}/benefits-summary.md`,
workingDirectory: benefitsDir,
maxTurns: 8,
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'plan-ceo-review-benefits',
runId,
});
@@ -634,7 +635,7 @@ Write your summary to ${benefitsDir}/benefits-summary.md`,
expect(summary).toMatch(/office.hours/);
expect(summary).toMatch(/design doc|no design/i);
}
}, 180_000);
}, CAPTURE_MS);
});
// --- Plan Review Report E2E ---
@@ -706,7 +707,7 @@ CRITICAL REQUIREMENT: plan.md IS the plan file for this review session. After co
This review report at the bottom of the plan is the MOST IMPORTANT deliverable of this test.`,
workingDirectory: planDir,
maxTurns: 20,
timeout: 360_000,
timeout: CAPTURE_LONG_MS,
testName: 'plan-review-report',
runId,
model: 'claude-opus-4-7',
@@ -768,7 +769,7 @@ This review report at the bottom of the plan is the MOST IMPORTANT deliverable o
).toBe(true);
console.log('Plan review report found at bottom of plan.md (ends with unresolved status)');
}, 420_000);
}, CAPTURE_LONG_MS);
});
// --- Codex Offering E2E ---
@@ -825,7 +826,7 @@ Summarize the Codex/${featureName} integration — answer these specific questio
Write your summary to ${testDir}/${testName}-summary.md`,
workingDirectory: testDir,
maxTurns: 8,
timeout: 120_000,
timeout: JUDGE_MS,
testName,
runId,
});
@@ -850,19 +851,19 @@ Write your summary to ${testDir}/${testName}-summary.md`,
testConcurrentIfSelected('codex-offered-office-hours', async () => {
await checkCodexOffering('office-hours', 'codex-offered-office-hours', 'second opinion');
}, 180_000);
}, CAPTURE_MS);
testConcurrentIfSelected('codex-offered-ceo-review', async () => {
await checkCodexOffering('plan-ceo-review', 'codex-offered-ceo-review', 'outside voice');
}, 180_000);
}, CAPTURE_MS);
testConcurrentIfSelected('codex-offered-design-review', async () => {
await checkCodexOffering('plan-design-review', 'codex-offered-design-review', 'design outside voices');
}, 180_000);
}, CAPTURE_MS);
testConcurrentIfSelected('codex-offered-eng-review', async () => {
await checkCodexOffering('plan-eng-review', 'codex-offered-eng-review', 'outside voice');
}, 180_000);
}, CAPTURE_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
@@ -40,6 +40,8 @@ const INLINE_REF = '29785978'; // last pre-Phase-1 commit (v1.69.1.0 bump)
function inlineSkill(): string {
return execSync(`git show ${INLINE_REF}:plan-ceo-review/SKILL.md`, {
// LIVE-REPO CWD: git show needs this repo's history to read the
// pre-Phase-1 SKILL.md render at INLINE_REF.
cwd: ROOT,
encoding: 'utf-8',
maxBuffer: 8 * 1024 * 1024,
+5 -4
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import { outcomeJudge } from './helpers/llm-judge';
import { judgePassed } from './helpers/eval-store';
@@ -97,7 +98,7 @@ CRITICAL RULES:
- The report MUST exist at ${reportPath} when you finish`,
workingDirectory: testWorkDir,
maxTurns: 50,
timeout: 300_000,
timeout: CAPTURE_MS,
testName: `qa-${label}`,
runId,
model: 'claude-opus-4-7',
@@ -174,17 +175,17 @@ CRITICAL RULES:
// B6: Static dashboard — broken link, disabled submit, overflow, missing alt, console error
testConcurrentIfSelected('qa-b6-static', async () => {
await runPlantedBugEval('qa-eval.html', 'qa-eval-ground-truth.json', 'b6-static');
}, 360_000);
}, CAPTURE_LONG_MS);
// B7: SPA — broken route, stale state, async race, missing aria, console warning
testConcurrentIfSelected('qa-b7-spa', async () => {
await runPlantedBugEval('qa-eval-spa.html', 'qa-eval-spa-ground-truth.json', 'b7-spa');
}, 360_000);
}, CAPTURE_LONG_MS);
// B8: Checkout — email regex, NaN total, CC overflow, missing required, stripe error
testConcurrentIfSelected('qa-b8-checkout', async () => {
await runPlantedBugEval('qa-eval-checkout.html', 'qa-eval-checkout-ground-truth.json', 'b8-checkout');
}, 360_000);
}, CAPTURE_LONG_MS);
});
+9 -8
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, browseBin, runId, evalsEnabled,
@@ -54,7 +55,7 @@ Do NOT try to start a server or discover ports — the URL above is ready.
Write your report to ${qaDir}/qa-reports/qa-report.md`,
workingDirectory: qaDir,
maxTurns: 35,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'qa-quick',
runId,
});
@@ -69,7 +70,7 @@ Write your report to ${qaDir}/qa-reports/qa-report.md`,
}
// Accept error_max_turns — the agent doing thorough QA work is not a failure
expect(['success', 'error_max_turns']).toContain(result.exitReason);
}, 300_000);
}, CAPTURE_MS);
});
// --- QA-Only E2E (report-only, no fixes) ---
@@ -124,7 +125,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`,
workingDirectory: qaOnlyDir,
maxTurns: 40,
allowedTools: ['Bash', 'Read', 'Write', 'Glob'], // NO Edit — the critical guardrail
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'qa-only-no-fix',
runId,
});
@@ -156,7 +157,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`,
(l: string) => l.trim() && !l.includes('.prompt-tmp') && !l.includes('.gstack/') && !l.includes('qa-reports/'),
);
expect(statusLines.filter((l: string) => l.startsWith(' M') || l.startsWith('M '))).toHaveLength(0);
}, 240_000);
}, CAPTURE_MS);
});
// --- QA Fix Loop E2E ---
@@ -247,7 +248,7 @@ This is a test+fix loop: find bugs, fix them in the source code, commit each fix
workingDirectory: qaFixDir,
maxTurns: 40,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
timeout: 420_000,
timeout: CAPTURE_LONG_MS,
testName: 'qa-fix-loop',
runId,
});
@@ -271,7 +272,7 @@ This is a test+fix loop: find bugs, fix them in the source code, commit each fix
// Verify Edit tool was used (agent actually modified source code)
const editCalls = result.toolCalls.filter(tc => tc.tool === 'Edit');
expect(editCalls.length).toBeGreaterThan(0);
}, 480_000);
}, CAPTURE_LONG_MS);
});
// --- Test Bootstrap E2E ---
@@ -384,7 +385,7 @@ Do NOT fix any bugs. Do NOT use AskUserQuestion — just pick vitest.`,
workingDirectory: bsDir,
maxTurns: 12,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'qa-bootstrap',
runId,
});
@@ -405,7 +406,7 @@ Do NOT fix any bugs. Do NOT use AskUserQuestion — just pick vitest.`,
console.log(`Test config: ${hasTestConfig}, Test file: ${hasTestFile}, TESTING.md: ${hasTestingMd}`);
try { fs.rmSync(bsDir, { recursive: true, force: true }); } catch {}
}, 120_000);
}, JUDGE_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete.
+4 -3
View File
@@ -1,4 +1,5 @@
import { expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId,
@@ -135,7 +136,7 @@ Write your retrospective to ${dir}/retro-output.md`,
expect(wroteReport).toBe(true);
const content = fs.readFileSync(retroPath, 'utf-8');
expect(content.length).toBeGreaterThan(100);
}, 480_000);
}, CAPTURE_LONG_MS);
});
// --- Retro E2E ---
@@ -198,7 +199,7 @@ Write your retrospective report to ${retroDir}/retro-output.md
Analyze the git history and produce the narrative report as described in the SKILL.md.`,
workingDirectory: retroDir,
maxTurns: 30,
timeout: 300_000,
timeout: CAPTURE_MS,
testName: 'retro',
runId,
model: 'claude-opus-4-7',
@@ -217,7 +218,7 @@ Analyze the git history and produce the narrative report as described in the SKI
expect(wroteReport).toBe(true);
const retro = fs.readFileSync(retroPath, 'utf-8');
expect(retro.length).toBeGreaterThan(100);
}, 420_000);
}, CAPTURE_LONG_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
+15 -14
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId, describeIfSelected, testConcurrentIfSelected,
@@ -114,7 +115,7 @@ and apply it yourself against the diff (git diff main...HEAD).
Write your findings to ${dir}/review-output.md`,
workingDirectory: dir,
maxTurns: 20,
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'review-army-migration-safety',
runId,
});
@@ -135,7 +136,7 @@ Write your findings to ${dir}/review-output.md`,
content.includes('column');
expect(hasMigrationFinding).toBe(true);
}
}, 210_000);
}, CAPTURE_MS);
});
// --- Review Army: N+1 Performance ---
@@ -179,7 +180,7 @@ For the specialist dispatch, read review-specialists/performance.md and apply it
Write your findings to ${dir}/review-output.md`,
workingDirectory: dir,
maxTurns: 20,
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'review-army-perf-n-plus-one',
runId,
});
@@ -201,7 +202,7 @@ Write your findings to ${dir}/review-output.md`,
content.includes('loop');
expect(hasN1Finding).toBe(true);
}
}, 210_000);
}, CAPTURE_MS);
});
// --- Review Army: Delivery Audit ---
@@ -281,7 +282,7 @@ The email notification system should be classified as NOT DONE.
Write your completion audit to ${dir}/review-output.md`,
workingDirectory: dir,
maxTurns: 15,
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'review-army-delivery-audit',
runId,
});
@@ -305,7 +306,7 @@ Write your completion audit to ${dir}/review-output.md`,
expect(hasNotDone).toBe(true);
expect(mentionsEmail).toBe(true);
}
}, 150_000);
}, CAPTURE_MS);
});
// --- Review Army: Quality Score ---
@@ -356,7 +357,7 @@ Write your findings AND the computed quality score to ${dir}/review-output.md
Include the line: "PR Quality Score: X/10" where X is the computed score.`,
workingDirectory: dir,
maxTurns: 15,
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'review-army-quality-score',
runId,
});
@@ -374,7 +375,7 @@ Include the line: "PR Quality Score: X/10" where X is the computed score.`,
content.match(/\d+\/10/);
expect(hasScore).toBeTruthy();
}
}, 150_000);
}, CAPTURE_MS);
});
// --- Review Army: JSON Findings ---
@@ -421,7 +422,7 @@ Output your findings as JSON objects, one per line, following the schema:
Write ONLY JSON findings (no preamble) to ${dir}/findings.json`,
workingDirectory: dir,
maxTurns: 12,
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'review-army-json-findings',
runId,
});
@@ -450,7 +451,7 @@ Write ONLY JSON findings (no preamble) to ${dir}/findings.json`,
break; // One valid line is enough for the gate test
}
}
}, 120_000);
}, JUDGE_MS);
});
// --- Review Army: Red Team (periodic) ---
@@ -499,7 +500,7 @@ Write your red team findings to ${dir}/review-output.md
Start the file with "RED TEAM REVIEW" on the first line.`,
workingDirectory: dir,
maxTurns: 20,
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'review-army-red-team',
runId,
});
@@ -513,7 +514,7 @@ Start the file with "RED TEAM REVIEW" on the first line.`,
const content = fs.readFileSync(outputPath, 'utf-8');
expect(content.toLowerCase()).toMatch(/red team|adversarial/);
}
}, 210_000);
}, CAPTURE_MS);
});
// --- Review Army: Consensus (periodic) ---
@@ -566,7 +567,7 @@ mark it as "MULTI-SPECIALIST CONFIRMED" with the confirming categories.
Write findings to ${dir}/review-output.md`,
workingDirectory: dir,
maxTurns: 20,
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'review-army-consensus',
runId,
});
@@ -585,7 +586,7 @@ Write findings to ${dir}/review-output.md`,
content.includes('interpolat');
expect(hasSqlFinding).toBe(true);
}
}, 210_000);
}, CAPTURE_MS);
});
// --- Review Army: Simplification specialist (activation) ---
+6 -5
View File
@@ -1,4 +1,5 @@
import { expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId,
@@ -65,7 +66,7 @@ Then run git diff against the detected base branch and write a brief review.
Write your findings to ${dir}/review-output.md`,
workingDirectory: dir,
maxTurns: 15,
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'review-base-branch',
runId,
});
@@ -84,7 +85,7 @@ Write your findings to ${dir}/review-output.md`,
return cmd.includes('git diff');
});
expect(usedGitDiff).toBe(true);
}, 120_000);
}, JUDGE_MS);
testConcurrentIfSelected('ship-base-branch', async () => {
const dir = path.join(baseBranchDir, 'ship-base');
@@ -125,7 +126,7 @@ Write a summary to ${dir}/ship-preflight.md including:
- The diff stat against the base branch`,
workingDirectory: dir,
maxTurns: 18,
timeout: 150_000,
timeout: CAPTURE_MS,
testName: 'ship-base-branch',
runId,
});
@@ -155,7 +156,7 @@ Write a summary to ${dir}/ship-preflight.md including:
return command.includes('git push') || command.includes('gh pr create');
});
expect(destructiveTools).toHaveLength(0);
}, 180_000);
}, CAPTURE_MS);
});
// --- Review Dashboard Via Attribution E2E ---
@@ -280,7 +281,7 @@ Write the dashboard output to ${dashDir}/dashboard-output.md`,
);
// Ship dashboard should not gate when eng review is clear
expect(gateQuestions).toHaveLength(0);
}, 480_000);
}, CAPTURE_LONG_MS);
});
// Module-level afterAll — finalize eval collector after all tests complete
+7 -6
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, browseBin, runId, evalsEnabled, selectedTests,
@@ -66,7 +67,7 @@ Run /review on the current diff (git diff main...HEAD).
Write your review findings to ${reviewDir}/review-output.md`,
workingDirectory: reviewDir,
maxTurns: 20,
timeout: 180_000,
timeout: CAPTURE_MS,
testName: 'review-sql-injection',
runId,
});
@@ -89,7 +90,7 @@ Write your review findings to ${reviewDir}/review-output.md`,
reviewContent.includes('unsanitized');
expect(hasSqlContent).toBe(true);
}
}, 210_000);
}, CAPTURE_MS);
});
// --- Review: Enum completeness E2E ---
@@ -144,7 +145,7 @@ Write your review findings to ${enumDir}/review-output.md
The diff adds a new "returned" status to the Order model. Your job is to check if all consumers handle it.`,
workingDirectory: enumDir,
maxTurns: 15,
timeout: 90_000,
timeout: JUDGE_MS,
testName: 'review-enum-completeness',
runId,
});
@@ -164,7 +165,7 @@ The diff adds a new "returned" status to the Order model. Your job is to check i
expect(mentionsReturned).toBe(true);
expect(mentionsEnum || mentionsCritical).toBe(true);
}
}, 120_000);
}, JUDGE_MS);
});
// --- Review: Design review lite E2E ---
@@ -229,7 +230,7 @@ Write your review findings to ${designDir}/review-output.md
Important: The design checklist should catch issues like blacklisted fonts, small font sizes, outline:none, !important, AI slop patterns (purple gradients, generic hero copy, 3-column feature grid), etc.`,
workingDirectory: designDir,
maxTurns: 35,
timeout: 240_000,
timeout: CAPTURE_MS,
testName: 'review-design-lite',
runId,
});
@@ -262,7 +263,7 @@ Important: The design checklist should catch issues like blacklisted fonts, smal
console.log(`Design review detected ${detected}/7 planted issues`);
expect(detected).toBeGreaterThanOrEqual(4);
}
}, 300_000);
}, CAPTURE_MS);
});
// Base branch detection tests for review/ship + the Review Dashboard Via
+54 -13
View File
@@ -1,4 +1,5 @@
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
ROOT, runId, evalsEnabled,
@@ -162,11 +163,12 @@ IMPORTANT:
Replace any references to ~/.claude/skills/gstack/bin/ with ./bin/ when running commands.
- Do NOT use AskUserQuestion.
- Just run the preamble bash block and report what you see.
- Look for "RECENT ARTIFACTS" and "LAST_SESSION" in the output.`,
- Look for "RECENT ARTIFACTS" and "LAST_SESSION" in the output.
- In your final message, quote VERBATIM (copy exactly, do not paraphrase) any output lines containing "RECENT ARTIFACTS" or "LAST_SESSION".`,
workingDirectory: workDir,
maxTurns: 10,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-recovery-artifacts',
runId,
});
@@ -193,7 +195,7 @@ IMPORTANT:
expect(foundCount).toBeGreaterThanOrEqual(1);
console.log(`Context recovery: artifacts=${foundArtifacts}, lastSession=${foundLastSession}, timeline=${foundTimeline}`);
}, 180_000);
}, CAPTURE_MS);
// --- Test 3: /context-save writes a file ---
// Hand-feed the save section of context-save/SKILL.md to claude -p and verify
@@ -231,7 +233,7 @@ IMPORTANT:
workingDirectory: workDir,
maxTurns: 10,
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-save-writes-file',
runId,
});
@@ -264,7 +266,7 @@ IMPORTANT:
expect(hasYamlFrontmatter).toBe(true);
console.log(`context-save: ${files.length} files created, YAML frontmatter: ${hasYamlFrontmatter}, branch: ${hasBranch}`);
}, 180_000);
}, CAPTURE_MS);
// --- Test 4: /context-restore loads the newest file across branches ---
// Seed two saved-context files with different YYYYMMDD-HHMMSS prefixes and
@@ -272,7 +274,17 @@ IMPORTANT:
// claude -p. Verify the agent identifies the newer file (by filename prefix)
// and presents its content, regardless of the current branch.
testConcurrentIfSelected('context-restore-loads-latest', async () => {
const projectDir = path.join(gstackHome, 'projects', slug);
// PRIVATE home for this test: the suite runs concurrently, and the shared
// gstackHome's checkpoints dir also receives the context-save test's
// freshly-written checkpoint (a 2026-08-29 filename prefix — always the
// "newest"). In CI, save completed before this test's agent listed the
// dir, so the agent CORRECTLY restored the sibling's checkpoint and the
// assertions failed; locally the ordering happened to run restore first.
// An isolated home makes the fixture set closed regardless of ordering
// (the agent may derive the dir from GSTACK_HOME/projects/<slug>, so the
// whole home moves, not just the checkpoint path we hand it).
const restoreHome = path.join(workDir, '.gstack-restore-home');
const projectDir = path.join(restoreHome, 'projects', slug);
const checkpointDir = path.join(projectDir, 'checkpoints');
fs.mkdirSync(checkpointDir, { recursive: true });
@@ -331,16 +343,19 @@ This is the newest saved context. Cross-branch restore should load THIS file.
${restoreSection.slice(0, 2500)}
IMPORTANT:
- Use GSTACK_HOME="${gstackHome}" as an environment variable when running bin scripts.
- Use GSTACK_HOME="${restoreHome}" as an environment variable when running bin scripts.
- The bin scripts are at ./bin/ (relative to this directory), not at ~/.claude/skills/gstack/bin/.
- Look in ${checkpointDir} for saved context files.
- Current branch is "main" do NOT filter by current branch. Load across all branches.
- The newest file by YYYYMMDD-HHMMSS prefix is the canonical "most recent". Filesystem mtime has been scrambled do not use it.
- Do NOT use AskUserQuestion. Just present the content of the newest file.`,
- Do NOT use AskUserQuestion. Just present the content of the newest file.
- Your final message MUST end with these two lines (they are machine-checked copy them exactly, do not paraphrase):
1. The newest file's "## Working on:" heading line, VERBATIM as it appears in that file.
2. A literal marker line: RESTORED: <filename of the newest file>`,
workingDirectory: workDir,
maxTurns: 8,
allowedTools: ['Bash', 'Read', 'Grep', 'Glob'],
timeout: 120_000,
timeout: JUDGE_MS,
testName: 'context-restore-loads-latest',
runId,
});
@@ -348,8 +363,34 @@ IMPORTANT:
logCost('context-restore', result);
const output = result.output ?? '';
const loadedNewer = output.includes('newer wintermute work') || output.includes('wintermute integration');
const loadedOlder = output.includes('old work') && !output.includes('newer');
// Evidence-based checks. CI receipts showed the agent finding + reading the
// RIGHT file, then paraphrasing the final message ("the most recent context
// is from branch-b...") — exact-substring checks over stochastic prose
// flaked. Three signal classes, strongest first:
// 1. Machine-checkable output contract (prompt demands the verbatim
// "## Working on:" heading + a "RESTORED: <filename>" marker line).
// 2. Lenient content echo (legacy): distinctive newer-file phrases.
// 3. Tool-call corroboration: a tool call whose INPUT names the newer
// file. Corroboration only — if the agent also read the OLDER file,
// tool evidence is void and the final output must present the newer.
const newerFileName = '20260202-130000-newer-wintermute-work';
const olderFileName = '20260101-120000-old-work';
const newerMarker = new RegExp(`RESTORED:.*${newerFileName}`, 'i').test(output);
const olderMarker = new RegExp(`RESTORED:.*${olderFileName}`, 'i').test(output);
const newerContent = output.includes('newer wintermute work') || output.includes('wintermute integration');
const outputPresentsNewer = newerMarker || newerContent;
const toolInputs = result.toolCalls.map(tc => JSON.stringify(tc.input ?? {}));
const toolReadNewer = toolInputs.some(input => input.includes(newerFileName));
const toolReadOlder = toolInputs.some(input => input.includes(olderFileName));
// Presenting the OLDER file fails: an explicit RESTORED marker naming it,
// or older-file content with no newer-file presentation alongside.
const loadedOlder = olderMarker || (output.includes('old work') && !outputPresentsNewer);
// Tool evidence counts only when the older file was never read: a run that
// reads BOTH files must present the NEWER one in the final output to pass.
const loadedNewer = outputPresentsNewer || (toolReadNewer && !toolReadOlder);
const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);
recordE2E(evalCollector, 'context-restore loads latest', 'Session Intelligence E2E', result, {
@@ -360,6 +401,6 @@ IMPORTANT:
expect(loadedNewer).toBe(true);
expect(loadedOlder).toBe(false);
console.log(`context-restore: loadedNewer=${loadedNewer}, loadedOlder=${loadedOlder}`);
}, 180_000);
console.log(`context-restore: loadedNewer=${loadedNewer} (marker=${newerMarker}, content=${newerContent}, toolNewer=${toolReadNewer}, toolOlder=${toolReadOlder}), loadedOlder=${loadedOlder}`);
}, CAPTURE_MS);
});

Some files were not shown because too many files have changed in this diff Show More