mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-12 07:59:02 +02:00
Merge remote-tracking branch 'origin/main' into ponytail-inspiration-review
# Conflicts: # CHANGELOG.md # TODOS.md # VERSION # docs/TESTING_INTERNALS.md # package.json # scripts/test-free-shards.ts # test/helpers/llm-judge.ts # test/helpers/touchfiles-data.ts
This commit is contained in:
@@ -13,6 +13,8 @@
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { spawnSync } from 'child_process';
|
||||
|
||||
import { runBin } from './helpers/run-bin';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
@@ -21,16 +23,15 @@ const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const BIN = path.join(ROOT, 'bin', 'gstack-model-benchmark');
|
||||
|
||||
function run(args: string[], opts: { env?: Record<string, string> } = {}): { status: number | null; stdout: string; stderr: string } {
|
||||
const result = spawnSync('bun', ['run', BIN, ...args], {
|
||||
const result = runBin('bun', ['run', BIN, ...args], {
|
||||
cwd: ROOT,
|
||||
env: { ...process.env, ...opts.env },
|
||||
encoding: 'utf-8',
|
||||
timeout: 15000,
|
||||
env: opts.env,
|
||||
timeoutMs: 15000,
|
||||
});
|
||||
return {
|
||||
status: result.status,
|
||||
stdout: result.stdout?.toString() ?? '',
|
||||
stderr: result.stderr?.toString() ?? '',
|
||||
stdout: result.stdout,
|
||||
stderr: result.stderr,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
/**
|
||||
* One Bun version across every CI surface.
|
||||
*
|
||||
* The drift class this pins: Dockerfile.ci's comment records that the old
|
||||
* `| BUN_VERSION=x.y.z bash` form silently installed latest on every image
|
||||
* rebuild (observed 1.3.13/1.3.14 drift vs the 1.3.10 devs ran locally),
|
||||
* and before 2026-08-29 the lanes disagreed four ways (1.3.13 / latest /
|
||||
* unpinned / 1.3.10). Different Bun versions change test-runner OUTPUT
|
||||
* SHAPES the strict classifiers regex-match, spawn semantics, and shell
|
||||
* parsing — a lane on a different Bun is testing a different product.
|
||||
*
|
||||
* Bumping Bun: change every surface in one commit; this test names each one.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
const WORKFLOWS_DIR = path.join(ROOT, '.github', 'workflows');
|
||||
|
||||
interface Pin {
|
||||
surface: string;
|
||||
version: string;
|
||||
}
|
||||
|
||||
function collectPins(): Pin[] {
|
||||
const pins: Pin[] = [];
|
||||
|
||||
for (const name of fs.readdirSync(WORKFLOWS_DIR).sort()) {
|
||||
if (!/\.ya?ml$/.test(name)) continue;
|
||||
const source = fs.readFileSync(path.join(WORKFLOWS_DIR, name), 'utf-8');
|
||||
const lines = source.split('\n');
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (!/uses:\s*oven-sh\/setup-bun@/.test(lines[i])) continue;
|
||||
// A pinned stanza is `with:` + `bun-version: <v>` within the next few
|
||||
// lines; an unpinned setup-bun is itself drift (installs latest).
|
||||
const window = lines.slice(i + 1, i + 4).join('\n');
|
||||
const m = window.match(/bun-version:\s*["']?([\w.]+)["']?/);
|
||||
pins.push({
|
||||
surface: `${name}:${i + 1}`,
|
||||
version: m ? m[1] : '<unpinned setup-bun — installs latest>',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const dockerfile = fs.readFileSync(
|
||||
path.join(ROOT, '.github', 'docker', 'Dockerfile.ci'), 'utf-8');
|
||||
const dockerPin = dockerfile.match(/bash -s ["']?bun-v([\w.]+)["']?/);
|
||||
pins.push({
|
||||
surface: 'Dockerfile.ci',
|
||||
version: dockerPin ? dockerPin[1] : '<no bun-vX.Y.Z positional arg>',
|
||||
});
|
||||
|
||||
const gitlab = fs.readFileSync(path.join(ROOT, '.gitlab-ci.yml'), 'utf-8');
|
||||
const gitlabPin = gitlab.match(/BUN_VERSION:\s*["']?([\w.]+)["']?/);
|
||||
pins.push({
|
||||
surface: '.gitlab-ci.yml',
|
||||
version: gitlabPin ? gitlabPin[1] : '<no BUN_VERSION>',
|
||||
});
|
||||
|
||||
return pins;
|
||||
}
|
||||
|
||||
describe('bun version pins', () => {
|
||||
test('every CI surface pins the same bun version', () => {
|
||||
const pins = collectPins();
|
||||
// Sanity: the scan found the known surfaces (a regex rot that finds
|
||||
// nothing must fail loudly, not vacuously pass).
|
||||
expect(pins.length).toBeGreaterThanOrEqual(6);
|
||||
|
||||
const versions = [...new Set(pins.map((p) => p.version))];
|
||||
const detail = pins.map((p) => `${p.surface} → ${p.version}`).join('\n');
|
||||
expect(versions, `bun version drift across CI surfaces:\n${detail}`).toHaveLength(1);
|
||||
expect(versions[0]).toMatch(/^\d+\.\d+\.\d+$/);
|
||||
});
|
||||
});
|
||||
@@ -20,6 +20,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { setupSkillDir, skillFromWorktree, captureSectionReads } from './helpers/auq-sdk-capture';
|
||||
import { CARVE_GUARDS } from './helpers/carve-guards';
|
||||
@@ -97,7 +98,7 @@ describeE2E('carve behavioral section-loading (periodic, SDK capture)', () => {
|
||||
});
|
||||
expect(output.trim().length).toBeGreaterThan(200);
|
||||
},
|
||||
540_000,
|
||||
CAPTURE_LONG_MS,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -15,14 +15,15 @@
|
||||
* `description: |` block (multi-line) instead of the trim'd one-line
|
||||
* `description: ...(gstack)` form.
|
||||
*
|
||||
* The smoke test mutates the working tree mid-run. It restores the default
|
||||
* trim'd state in a finally block so a crash mid-test still leaves a clean
|
||||
* working tree.
|
||||
* The smoke test renders the full-catalog variant into an isolated
|
||||
* --out-dir — the working tree is never written, so there is no restore
|
||||
* pass (and no half-restored tree if the test crashes mid-run).
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
|
||||
const REPO_ROOT = path.resolve(import.meta.dir, '..');
|
||||
@@ -58,24 +59,26 @@ describe('--catalog-mode=full opt-out wiring (static)', () => {
|
||||
|
||||
describe('--catalog-mode=full opt-out behavior (smoke)', () => {
|
||||
test('--catalog-mode=full produces multi-line description in frontmatter', () => {
|
||||
// Save the trim'd state so we can restore it.
|
||||
const trimmedShip = fs.readFileSync(SHIP_SKILL, 'utf-8');
|
||||
// The TRACKED ship/SKILL.md carries the default trim'd form (read-only check).
|
||||
// #1778: the trimmed ship description has an interior colon ("Ship workflow:")
|
||||
// and is now YAML-quoted — tolerate the optional surrounding quotes.
|
||||
const trimmedShip = fs.readFileSync(SHIP_SKILL, 'utf-8');
|
||||
expect(trimmedShip).toMatch(/^description: "?Ship workflow:[^\n]*\(gstack\)"?\n/m);
|
||||
|
||||
const outDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-catalog-full-'));
|
||||
try {
|
||||
// Run with --catalog-mode=full. Mutates working tree.
|
||||
const result = spawnSync('bun', ['run', 'gen:skill-docs', '--catalog-mode=full'], {
|
||||
// Render --catalog-mode=full into an isolated out-dir. The working
|
||||
// tree is never written, so no restore pass is needed.
|
||||
const result = spawnSync('bun', ['run', 'gen:skill-docs', '--catalog-mode=full', '--out-dir', outDir], {
|
||||
cwd: REPO_ROOT,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
timeout: 60_000,
|
||||
});
|
||||
expect(result.status).toBe(0);
|
||||
|
||||
// After --catalog-mode=full, frontmatter description is the legacy
|
||||
// In the full-mode render, frontmatter description is the legacy
|
||||
// multi-line block, not the trim'd one-line form.
|
||||
const fullShip = fs.readFileSync(SHIP_SKILL, 'utf-8');
|
||||
const fullShip = fs.readFileSync(path.join(outDir, 'ship', 'SKILL.md'), 'utf-8');
|
||||
expect(fullShip).toMatch(/^description: \|\s*$/m); // YAML block scalar
|
||||
// Legacy multi-line content includes "Use when asked to..." in the
|
||||
// frontmatter (in trim mode this lives in the body section).
|
||||
@@ -87,23 +90,12 @@ describe('--catalog-mode=full opt-out behavior (smoke)', () => {
|
||||
// (because the routing prose stayed in frontmatter).
|
||||
const body = fullShip.slice(fmEnd);
|
||||
expect(body).not.toContain('## When to invoke this skill');
|
||||
|
||||
// Non-mutation proof: the tracked ship/SKILL.md is byte-unchanged —
|
||||
// a catalog-mode render must never rewrite the committed trim'd state.
|
||||
expect(fs.readFileSync(SHIP_SKILL, 'utf-8')).toBe(trimmedShip);
|
||||
} finally {
|
||||
// Restore default trim state regardless of test outcome.
|
||||
const restore = spawnSync('bun', ['run', 'gen:skill-docs'], {
|
||||
cwd: REPO_ROOT,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
timeout: 60_000,
|
||||
});
|
||||
if (restore.status !== 0) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(
|
||||
'CRITICAL: failed to restore default trim state. Run `bun run gen:skill-docs` to clean up.',
|
||||
);
|
||||
}
|
||||
// Sanity-check the restored state matches what we saw at the start.
|
||||
const restoredShip = fs.readFileSync(SHIP_SKILL, 'utf-8');
|
||||
// #1778: restored trim state has the YAML-quoted (interior-colon) description.
|
||||
expect(restoredShip).toMatch(/^description: "?Ship workflow:[^\n]*\(gstack\)"?\n/m);
|
||||
fs.rmSync(outDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
/**
|
||||
* The CI image tag is a content hash computed independently in three
|
||||
* workflows — evals.yml, evals-periodic.yml, ci-image.yml — and they were
|
||||
* synced by comment only (filed in TODOS.md as the "three-way image-tag
|
||||
* drift" gap). If one file's hashFiles() input list drifts, that workflow
|
||||
* computes a DIFFERENT tag for the same content: the eval lanes stop finding
|
||||
* the prebuilt image and silently rebuild it on every run (minutes per run,
|
||||
* no red check), or ci-image prebuilds a tag nobody looks up.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
const FILES = ['evals.yml', 'evals-periodic.yml', 'ci-image.yml'];
|
||||
|
||||
function hashFilesCalls(name: string): string[] {
|
||||
const source = fs.readFileSync(
|
||||
path.join(ROOT, '.github', 'workflows', name), 'utf-8');
|
||||
// Only tag-computation sites: hashFiles() inside a `tag=` output line.
|
||||
return [...source.matchAll(/tag=[^\n]*?(hashFiles\([^)]*\))/g)].map((m) => m[1]);
|
||||
}
|
||||
|
||||
describe('ci image tag binding', () => {
|
||||
test('all three workflows compute the tag from the identical hashFiles() input list', () => {
|
||||
const perFile = FILES.map((f) => ({ file: f, calls: hashFilesCalls(f) }));
|
||||
for (const { file, calls } of perFile) {
|
||||
// Each workflow computes the tag exactly once; zero means the scan
|
||||
// regex rotted (must fail loudly, not vacuously pass).
|
||||
expect(calls, `${file}: expected exactly one tag hashFiles() site`).toHaveLength(1);
|
||||
}
|
||||
const expressions = [...new Set(perFile.map((p) => p.calls[0]))];
|
||||
const detail = perFile.map((p) => `${p.file} → ${p.calls[0]}`).join('\n');
|
||||
expect(expressions, `image-tag hashFiles() drift:\n${detail}`).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,205 @@
|
||||
/**
|
||||
* bin/gstack-code-intelligence — CLI surface smoke tests.
|
||||
*
|
||||
* lib/code-intelligence/* is covered by test/code-intelligence.test.ts, which
|
||||
* also drives the CLI's `index` and `search` consent/policy refusal paths.
|
||||
* This file covers the argument-handling surface those tests skip: usage on
|
||||
* bad/missing subcommands, `select` and `consent` validation + state writes,
|
||||
* and the `suggest` offer gate — all hermetic under a mkdtemp GSTACK_HOME
|
||||
* (the selection store lives at $GSTACK_HOME/code-intelligence.json), and all
|
||||
* on paths that never call detectAvailable(), so nothing probes providers or
|
||||
* the network.
|
||||
*
|
||||
* Note: the CLI has no `--help` flag — every unrecognized action (including
|
||||
* `--help`) routes to the usage message on stderr with exit 1. Pinned below.
|
||||
*/
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { runBin } from './helpers/run-bin';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const CLI = path.join(ROOT, 'bin', 'gstack-code-intelligence');
|
||||
|
||||
let home: string;
|
||||
let workDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
home = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-cli-home-'));
|
||||
workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-cli-work-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(home, { recursive: true, force: true });
|
||||
fs.rmSync(workDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function runCli(...args: string[]) {
|
||||
return runBin('bun', [CLI, ...args], { cwd: workDir, gstackHome: home, home });
|
||||
}
|
||||
|
||||
function readStore(): { provider: string | null; consents: Record<string, boolean>; declined: boolean } {
|
||||
return JSON.parse(fs.readFileSync(path.join(home, 'code-intelligence.json'), 'utf-8'));
|
||||
}
|
||||
|
||||
describe('gstack-code-intelligence: usage surface', () => {
|
||||
test('no arguments: usage on stderr, exit 1', () => {
|
||||
const result = runCli();
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain('gstack-code-intelligence:');
|
||||
expect(result.stderr).toContain('Usage:');
|
||||
expect(result.stderr).toContain('select <provider>');
|
||||
expect(result.stdout).toBe('');
|
||||
});
|
||||
|
||||
test('unknown subcommand: usage on stderr, exit 1', () => {
|
||||
const result = runCli('frobnicate');
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain('Usage:');
|
||||
});
|
||||
|
||||
test('--help has no exit-0 handler — it routes to the usage failure (current behavior)', () => {
|
||||
const result = runCli('--help');
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain('Usage:');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-code-intelligence: select', () => {
|
||||
test('invalid provider is rejected with the select usage line', () => {
|
||||
const result = runCli('select', 'bogus-provider');
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain('Usage: select <gbrain|sourcebot|graphify|none>');
|
||||
expect(fs.existsSync(path.join(home, 'code-intelligence.json'))).toBe(false);
|
||||
});
|
||||
|
||||
test('select with no argument is rejected the same way', () => {
|
||||
const result = runCli('select');
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain('Usage: select <gbrain|sourcebot|graphify|none>');
|
||||
});
|
||||
|
||||
test('select none records the decline so the offer is never repeated', () => {
|
||||
const result = runCli('select', 'none');
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain('declined');
|
||||
expect(result.stdout).toContain('will not ask again');
|
||||
const store = readStore();
|
||||
expect(store.provider).toBeNull();
|
||||
expect(store.declined).toBe(true);
|
||||
});
|
||||
|
||||
test('selecting the local provider persists it without an off-machine warning', () => {
|
||||
const result = runCli('select', 'graphify');
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain('selected Graphify.');
|
||||
expect(result.stdout).not.toContain('off this machine');
|
||||
const store = readStore();
|
||||
expect(store.provider).toBe('graphify');
|
||||
expect(store.declined).toBe(false);
|
||||
});
|
||||
|
||||
test('selecting a non-local provider warns that content leaves the machine', () => {
|
||||
const result = runCli('select', 'gbrain');
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain('selected GBrain.');
|
||||
expect(result.stdout).toContain('off this machine');
|
||||
expect(readStore().provider).toBe('gbrain');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-code-intelligence: consent', () => {
|
||||
test('the yes/no value is required — a bare path records NOTHING', () => {
|
||||
const result = runCli('consent', workDir);
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain('never assumed');
|
||||
expect(fs.existsSync(path.join(home, 'code-intelligence.json'))).toBe(false);
|
||||
});
|
||||
|
||||
test('an unknown value records NOTHING', () => {
|
||||
const result = runCli('consent', workDir, 'maybe');
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain('never assumed');
|
||||
expect(fs.existsSync(path.join(home, 'code-intelligence.json'))).toBe(false);
|
||||
});
|
||||
|
||||
test('consent yes persists true for the resolved repo path', () => {
|
||||
const result = runCli('consent', workDir, 'yes');
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain('indexing consent recorded');
|
||||
expect(readStore().consents[fs.realpathSync(workDir)] ?? readStore().consents[workDir]).toBe(true);
|
||||
});
|
||||
|
||||
test('consent no persists an explicit DENIED — a "no" is a durable answer too', () => {
|
||||
const result = runCli('consent', workDir, 'no');
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain('DENIED');
|
||||
expect(readStore().consents[fs.realpathSync(workDir)] ?? readStore().consents[workDir]).toBe(false);
|
||||
});
|
||||
|
||||
test('consent with no path defaults to the cwd', () => {
|
||||
const result = runCli('consent', 'yes');
|
||||
expect(result.status).toBe(0);
|
||||
const consents = readStore().consents;
|
||||
const keys = Object.keys(consents);
|
||||
expect(keys.length).toBe(1);
|
||||
// resolve(cwd) — the child's cwd is workDir (possibly via a symlinked tmp).
|
||||
expect([workDir, fs.realpathSync(workDir)]).toContain(keys[0]);
|
||||
expect(consents[keys[0]]).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-code-intelligence: suggest (offer gate)', () => {
|
||||
test('a non-repo directory never triggers the offer (--json)', () => {
|
||||
const result = runCli('suggest', workDir, '--json');
|
||||
expect(result.status).toBe(0);
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
expect(parsed.offer).toBe(false);
|
||||
expect(parsed.reason).toBe('not-a-repo');
|
||||
expect(parsed.fileCount).toBeNull();
|
||||
expect([workDir, fs.realpathSync(workDir)]).toContain(parsed.repoPath);
|
||||
});
|
||||
|
||||
test('a selected provider suppresses the offer before any repo probing', () => {
|
||||
expect(runCli('select', 'graphify').status).toBe(0);
|
||||
const result = runCli('suggest', workDir, '--json');
|
||||
expect(result.status).toBe(0);
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
expect(parsed.offer).toBe(false);
|
||||
expect(parsed.reason).toBe('provider-selected');
|
||||
});
|
||||
|
||||
test('an explicit decline suppresses the offer permanently', () => {
|
||||
expect(runCli('select', 'none').status).toBe(0);
|
||||
const result = runCli('suggest', workDir, '--json');
|
||||
expect(result.status).toBe(0);
|
||||
expect(JSON.parse(result.stdout).reason).toBe('declined');
|
||||
});
|
||||
|
||||
test('human-readable no-offer output names the reason', () => {
|
||||
const result = runCli('suggest', workDir);
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain('no offer (not-a-repo)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gstack-code-intelligence: provider-requiring commands without a selection', () => {
|
||||
test('index refuses when no provider is selected', () => {
|
||||
const result = runCli('index', workDir);
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain('no provider selected');
|
||||
});
|
||||
|
||||
test('search refuses when no provider is selected', () => {
|
||||
const result = runCli('search', 'anything');
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain('no provider selected');
|
||||
});
|
||||
|
||||
test('search with no query prints the search usage', () => {
|
||||
const result = runCli('search');
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain('Usage: search <query...>');
|
||||
});
|
||||
});
|
||||
@@ -26,6 +26,7 @@
|
||||
* Periodic tier (Codex non-determinism). Cost: ~$2-3 per full run.
|
||||
*/
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runCodexSkill, installSkillToTempHome } from './helpers/codex-session-runner';
|
||||
import type { CodexResult } from './helpers/codex-session-runner';
|
||||
import { EvalCollector } from './helpers/eval-store';
|
||||
@@ -47,7 +48,12 @@ const CODEX_AVAILABLE = (() => {
|
||||
} catch { return false; }
|
||||
})();
|
||||
const evalsEnabled = !!process.env.EVALS;
|
||||
const SKIP = !CODEX_AVAILABLE || !evalsEnabled;
|
||||
// External-service test — periodic tier only (CLAUDE.md tiering rule 3),
|
||||
// matching codex-e2e.test.ts / codex-e2e-sol-scope.test.ts. Without this
|
||||
// guard the sharded runner's "no whole-file tier guard" default would run
|
||||
// Codex spawns in the GATE tier on every PR.
|
||||
const tierOk = process.env.EVALS_TIER === 'periodic';
|
||||
const SKIP = !CODEX_AVAILABLE || !evalsEnabled || !tierOk;
|
||||
const describeCodex = SKIP ? describe.skip : describe;
|
||||
|
||||
// --- Touchfiles ---
|
||||
@@ -181,7 +187,7 @@ describeCodex('Codex Plan Format — CEO Mode Selection', () => {
|
||||
const result = await runCodexSkill({
|
||||
skillDir,
|
||||
prompt: `Read the plan-ceo-review skill. Read plan.md (the plan to review). Proceed to Step 0F (Mode Selection) where the skill presents 4 mode options (SCOPE EXPANSION, SELECTIVE EXPANSION, HOLD SCOPE, SCOPE REDUCTION) via AskUserQuestion. These options differ in kind (review posture), not coverage. ${captureInstruction(outFile)}`,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
cwd: planDir,
|
||||
skillName: 'gstack-plan-ceo-review',
|
||||
sandbox: 'workspace-write',
|
||||
@@ -203,7 +209,7 @@ describeCodex('Codex Plan Format — CEO Mode Selection', () => {
|
||||
// kind-differentiated: no fabricated score, must have note
|
||||
expect(captured).not.toMatch(COMPLETENESS_RE);
|
||||
expect(captured).toMatch(KIND_NOTE_RE);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
describeCodex('Codex Plan Format — CEO Approach Menu', () => {
|
||||
@@ -221,7 +227,7 @@ describeCodex('Codex Plan Format — CEO Approach Menu', () => {
|
||||
const result = await runCodexSkill({
|
||||
skillDir,
|
||||
prompt: `Read the plan-ceo-review skill. Read plan.md. Proceed to Step 0C-bis (Implementation Alternatives / Approach Menu) where the skill generates 2-3 approaches (minimal viable vs ideal architecture) and presents them via AskUserQuestion. These options differ in coverage so Completeness: N/10 applies. ${captureInstruction(outFile)}`,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
cwd: planDir,
|
||||
skillName: 'gstack-plan-ceo-review',
|
||||
sandbox: 'workspace-write',
|
||||
@@ -240,7 +246,7 @@ describeCodex('Codex Plan Format — CEO Approach Menu', () => {
|
||||
expect(captured.length).toBeGreaterThan(ELI10_LENGTH_FLOOR);
|
||||
expect(captured).toMatch(RECOMMENDATION_RE);
|
||||
expect(captured).toMatch(COMPLETENESS_RE);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
describeCodex('Codex Plan Format — Eng Coverage Issue', () => {
|
||||
@@ -258,7 +264,7 @@ describeCodex('Codex Plan Format — Eng Coverage Issue', () => {
|
||||
const result = await runCodexSkill({
|
||||
skillDir,
|
||||
prompt: `Read the plan-eng-review skill. Read plan.md. In your Section 3 Test Review, generate ONE AskUserQuestion about test coverage depth where options are clearly coverage-differentiated: A) full coverage incl. edge + error paths (Completeness 10/10), B) happy path only (7/10), C) smoke test (3/10). ${captureInstruction(outFile)}`,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
cwd: planDir,
|
||||
skillName: 'gstack-plan-eng-review',
|
||||
sandbox: 'workspace-write',
|
||||
@@ -277,7 +283,7 @@ describeCodex('Codex Plan Format — Eng Coverage Issue', () => {
|
||||
expect(captured.length).toBeGreaterThan(ELI10_LENGTH_FLOOR);
|
||||
expect(captured).toMatch(RECOMMENDATION_RE);
|
||||
expect(captured).toMatch(COMPLETENESS_RE);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
describeCodex('Codex Plan Format — Eng Kind Issue', () => {
|
||||
@@ -295,7 +301,7 @@ describeCodex('Codex Plan Format — Eng Kind Issue', () => {
|
||||
const result = await runCodexSkill({
|
||||
skillDir,
|
||||
prompt: `Read the plan-eng-review skill. Read plan.md. In your Section 1 Architecture review, generate ONE AskUserQuestion about an architectural choice where the options differ in kind (e.g. Redis vs Postgres materialized view vs in-process cache — different kinds of systems with different tradeoffs, NOT more-or-less-complete versions of the same thing). ${captureInstruction(outFile)}`,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
cwd: planDir,
|
||||
skillName: 'gstack-plan-eng-review',
|
||||
sandbox: 'workspace-write',
|
||||
@@ -316,5 +322,5 @@ describeCodex('Codex Plan Format — Eng Kind Issue', () => {
|
||||
// kind-differentiated: no fabricated score
|
||||
expect(captured).not.toMatch(COMPLETENESS_RE);
|
||||
expect(captured).toMatch(KIND_NOTE_RE);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
* Periodic tier (Codex non-determinism, ~$2-3/run).
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import * as path from 'node:path';
|
||||
import { e2eTierEnabled } from './helpers/e2e-gate';
|
||||
import { runCodexSkill } from './helpers/codex-session-runner';
|
||||
@@ -69,7 +70,7 @@ describeCodex('/codex recommendation substance (live, periodic)', () => {
|
||||
skillDir: path.join(ROOT, 'codex'),
|
||||
skillName: 'codex',
|
||||
prompt: FIXTURE_DIFF,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (result.output.startsWith('SKIP:')) {
|
||||
@@ -98,6 +99,6 @@ describeCodex('/codex recommendation substance (live, periodic)', () => {
|
||||
);
|
||||
}
|
||||
},
|
||||
360_000,
|
||||
CAPTURE_LONG_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
* golden), parallel shards (worktree copies), or live symlinked installs.
|
||||
*/
|
||||
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
@@ -125,6 +126,9 @@ describeSol('GPT-5.6 Sol full-artifact scope termination', () => {
|
||||
const generated = spawnSync(
|
||||
'bun',
|
||||
['run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--model', 'gpt-5.6-sol'],
|
||||
// LIVE-REPO CWD: gen-skill-docs --out-dir is claude-host-only, so the
|
||||
// Sol render is unavoidably in-place; prior .agents tree is snapshotted
|
||||
// above and restored below.
|
||||
{ cwd: ROOT, encoding: 'utf8', timeout: 120_000 },
|
||||
);
|
||||
if (generated.status !== 0) {
|
||||
@@ -259,5 +263,5 @@ You are authorized to implement the minimal fix. The task boundary is src/parse-
|
||||
expect(readmeDecoyUntouched).toBe(true);
|
||||
|
||||
console.log(`codex-sol-scope: ${result.tokens} tokens, ${result.toolCalls.length} tool calls, ${Math.round(result.durationMs / 1000)}s`);
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runCodexSkill, parseCodexJSONL, installSkillToTempHome } from './helpers/codex-session-runner';
|
||||
import type { CodexResult } from './helpers/codex-session-runner';
|
||||
import { CODEX_REVIEW_E2E_SECTIONS } from './helpers/skill-fixture';
|
||||
@@ -150,7 +151,7 @@ describeCodex('Codex E2E', () => {
|
||||
const result = await runCodexSkill({
|
||||
skillDir,
|
||||
prompt: 'List any skills or instructions you have available. Just list the names.',
|
||||
timeoutMs: 60_000,
|
||||
timeoutMs: JUDGE_MS,
|
||||
cwd: testWorktree,
|
||||
skillName: 'gstack-review',
|
||||
});
|
||||
@@ -171,7 +172,7 @@ describeCodex('Codex E2E', () => {
|
||||
expect(
|
||||
outputLower.includes('review') || outputLower.includes('gstack') || outputLower.includes('skill'),
|
||||
).toBe(true);
|
||||
}, 120_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
// Validates that Codex can invoke the gstack-review skill, run a diff-based
|
||||
// code review, and produce structured review output with findings/issues.
|
||||
@@ -186,7 +187,7 @@ describeCodex('Codex E2E', () => {
|
||||
const result = await runCodexSkill({
|
||||
skillDir,
|
||||
prompt: 'Run the gstack-review skill on this repository. Review the current branch diff and report your findings.',
|
||||
timeoutMs: 540_000,
|
||||
timeoutMs: CAPTURE_LONG_MS,
|
||||
cwd: testWorktree,
|
||||
skillName: 'gstack-review',
|
||||
sections: CODEX_REVIEW_E2E_SECTIONS,
|
||||
@@ -224,5 +225,5 @@ describeCodex('Codex E2E', () => {
|
||||
outputLower.includes('p1') ||
|
||||
outputLower.includes('p2');
|
||||
expect(hasReviewContent).toBe(true);
|
||||
}, 600_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -15,47 +15,31 @@ import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const SKILL_GLOBS = [
|
||||
'plan-ceo-review',
|
||||
'plan-eng-review',
|
||||
'plan-design-review',
|
||||
'plan-devex-review',
|
||||
'office-hours',
|
||||
'codex',
|
||||
'investigate',
|
||||
'qa',
|
||||
'retro',
|
||||
'cso',
|
||||
'review',
|
||||
'ship',
|
||||
'design-review',
|
||||
'devex-review',
|
||||
'qa-only',
|
||||
'design-consultation',
|
||||
'design-shotgun',
|
||||
'autoplan',
|
||||
'land-and-deploy',
|
||||
'plan-tune',
|
||||
'document-release',
|
||||
'context-save',
|
||||
'context-restore',
|
||||
'health',
|
||||
'setup-deploy',
|
||||
'setup-browser-cookies',
|
||||
'canary',
|
||||
'learn',
|
||||
'benchmark',
|
||||
'benchmark-models',
|
||||
'make-pdf',
|
||||
'open-gstack-browser',
|
||||
'gstack-upgrade',
|
||||
'pair-agent',
|
||||
'design-html',
|
||||
'freeze',
|
||||
'unfreeze',
|
||||
'careful',
|
||||
'guard',
|
||||
];
|
||||
|
||||
/**
|
||||
* Every top-level skill directory with a SKILL.md.tmpl, discovered from
|
||||
* disk. This replaced a hand-maintained 39-name list that had drifted to
|
||||
* 39-of-54 templates on disk — none of the unlisted 15 happened to be
|
||||
* interactive, so there was no LIVE gap, but the next interactive skill
|
||||
* would have landed unguarded with no signal.
|
||||
*/
|
||||
function skillTemplateDirs(): string[] {
|
||||
return fs.readdirSync(ROOT, { withFileTypes: true })
|
||||
.filter((entry) => {
|
||||
// Directory symlinks (connect-chrome → open-gstack-browser) count too:
|
||||
// existsSync below follows them, and duplicates only re-check the same
|
||||
// template. isDirectory() is false for symlinked dirs, hence statSync.
|
||||
if (entry.name.startsWith('.') || entry.name === 'node_modules') return false;
|
||||
try {
|
||||
return fs.statSync(path.join(ROOT, entry.name)).isDirectory()
|
||||
&& fs.existsSync(path.join(ROOT, entry.name, 'SKILL.md.tmpl'));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})
|
||||
.map((entry) => entry.name)
|
||||
.sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* Load .tmpl files for each skill and return the names of those that have
|
||||
@@ -63,7 +47,7 @@ const SKILL_GLOBS = [
|
||||
*/
|
||||
function findInteractiveSkills(): string[] {
|
||||
const interactive: string[] = [];
|
||||
for (const skill of SKILL_GLOBS) {
|
||||
for (const skill of skillTemplateDirs()) {
|
||||
const tmplPath = path.join(ROOT, skill, 'SKILL.md.tmpl');
|
||||
if (!fs.existsSync(tmplPath)) continue;
|
||||
const content = fs.readFileSync(tmplPath, 'utf-8');
|
||||
|
||||
@@ -11,9 +11,15 @@
|
||||
* Mapping rule (test filenames do NOT map mechanically to tier keys): for each
|
||||
* `test/skill-e2e-*.test.ts` with an EVALS_TIER self-gate, search the
|
||||
* E2E_TOUCHFILES / LLM_JUDGE_TOUCHFILES dep lists for the exact file path. If
|
||||
* found under key K, the file's self-gate tier must equal E2E_TIERS[K]. Files
|
||||
* not named in any dep list are REPORTED as unmapped (a nudge to add them to
|
||||
* their eval's dep list), never silently skipped.
|
||||
* found under key K, the file's self-gate tier must equal E2E_TIERS[K].
|
||||
*
|
||||
* Self-registration is a HARD invariant (the dep-list sweep): every
|
||||
* skill-e2e file must be named in at least one touchfiles dep list, so that
|
||||
* editing only the test's prompt/assertions diff-selects the test itself.
|
||||
* Before the sweep, 129 of ~177 E2E keys did not list their own declaring
|
||||
* file — a changed test never re-ran on its own change. Files that genuinely
|
||||
* cannot be mapped (no E2E map key exists for them) sit in KNOWN_UNREGISTERED
|
||||
* below; that set is a ratchet, it only shrinks.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
@@ -35,6 +41,26 @@ const SELF_GATE_RE = /EVALS_TIER\s*===\s*['"](gate|periodic)['"]/g;
|
||||
// the declared tier exactly like the raw predicate's tier literal did.
|
||||
const HELPER_GATE_RE = /\b(?:describeE2ETier|e2eTierEnabled)\(\s*['"](gate|periodic)['"]/g;
|
||||
|
||||
/**
|
||||
* Ratchet, not amnesty (same contract as KNOWN_MATRIX_GAPS in
|
||||
* test/evals-workflow-matrix.test.ts): skill-e2e files that are named in NO
|
||||
* touchfiles dep list because no E2E map key exists for them. Every entry
|
||||
* carries a one-line reason. Do NOT add new files here — give the test an
|
||||
* E2E map key (touchfiles + tier) and register the file in its dep list.
|
||||
* A stale entry (file deleted, or file now registered) FAILS the suite —
|
||||
* delete it. Target: empty set.
|
||||
*/
|
||||
const KNOWN_UNREGISTERED = new Set([
|
||||
// Standalone periodic self-gated probe; template-literal testNames (auq-consistency-${i}), no E2E map key — fail-open-safe, runs on every periodic sweep.
|
||||
'test/skill-e2e-auq-consistency.test.ts',
|
||||
// Standalone periodic self-gated matrix; template-literal testNames (auq-matrix-${m.skill}), no E2E map key — fail-open-safe, runs on every periodic sweep.
|
||||
'test/skill-e2e-auq-matrix.test.ts',
|
||||
// Standalone periodic self-gated A/B probe; template-literal testNames (auq-ab-${label}), no E2E map key — fail-open-safe, runs on every periodic sweep.
|
||||
'test/skill-e2e-auq-verbose-vs-carved-ab.test.ts',
|
||||
// bin-script pipeline test (spawns bun scripts, no model spend) that lives under the skill-e2e-* glob; no E2E map key exists for it.
|
||||
'test/skill-e2e-memory-pipeline.test.ts',
|
||||
]);
|
||||
|
||||
describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () => {
|
||||
const testFiles = readdirSync(TEST_DIR)
|
||||
.filter((f) => f.startsWith('skill-e2e-') && f.endsWith('.test.ts'))
|
||||
@@ -44,14 +70,28 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () =>
|
||||
|
||||
test('every self-gated test file named in a dep list matches its declared tier', () => {
|
||||
const misaligned: string[] = [];
|
||||
const unregistered: string[] = [];
|
||||
const reported: string[] = [];
|
||||
|
||||
for (const file of testFiles) {
|
||||
const content = readFileSync(path.join(TEST_DIR, file), 'utf-8');
|
||||
const repoPath = `test/${file}`;
|
||||
|
||||
// HARD self-registration invariant, independent of self-gate shape:
|
||||
// a skill-e2e file named in no dep list means editing the test itself
|
||||
// selects nothing — the changed test never re-runs on its own change.
|
||||
const owningKeys = Object.keys(allDeps).filter((k) => allDeps[k].includes(repoPath));
|
||||
if (owningKeys.length === 0 && !KNOWN_UNREGISTERED.has(repoPath)) {
|
||||
unregistered.push(
|
||||
`${repoPath}: not named in any touchfiles dep list — editing this test file would `
|
||||
+ 'never diff-select it. Add the file path to its E2E map key\'s dep list in '
|
||||
+ 'test/helpers/touchfiles-data.ts (do NOT extend KNOWN_UNREGISTERED for new files).',
|
||||
);
|
||||
}
|
||||
|
||||
const tiers = new Set<string>();
|
||||
for (const m of content.matchAll(SELF_GATE_RE)) tiers.add(m[1]);
|
||||
for (const m of content.matchAll(HELPER_GATE_RE)) tiers.add(m[1]);
|
||||
const repoPath = `test/${file}`;
|
||||
if (tiers.size === 0) {
|
||||
// Every skill-e2e file is expected to self-gate; zero matches means
|
||||
// either a genuinely ungated file or a gate shape the regex can't
|
||||
@@ -65,8 +105,9 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () =>
|
||||
}
|
||||
const selfTier = [...tiers][0];
|
||||
|
||||
const owningKeys = Object.keys(allDeps).filter((k) => allDeps[k].includes(repoPath));
|
||||
if (owningKeys.length === 0) {
|
||||
// Only KNOWN_UNREGISTERED files reach here (anything else already
|
||||
// hard-failed above) — keep the visible nudge.
|
||||
reported.push(`${repoPath} (self-gates '${selfTier}'): not named in any touchfiles dep list`);
|
||||
continue;
|
||||
}
|
||||
@@ -86,18 +127,36 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () =>
|
||||
}
|
||||
}
|
||||
|
||||
// Reported, not asserted: coverage holes the invariant can see but not
|
||||
// arbitrate. Add the test file to its eval's dep list (or a tier entry
|
||||
// for the key) to bring it under the invariant.
|
||||
// Reported, not asserted: tier-observability holes the invariant can see
|
||||
// but not arbitrate (map-driven files legitimately have no whole-file
|
||||
// self-gate; ratcheted files stay visible). Registration itself is
|
||||
// asserted below.
|
||||
if (reported.length > 0) {
|
||||
console.warn(
|
||||
`[tier-alignment] ${reported.length} file(s) outside the invariant:\n ` + reported.join('\n '),
|
||||
`[tier-alignment] ${reported.length} file(s) outside the tier invariant:\n ` + reported.join('\n '),
|
||||
);
|
||||
}
|
||||
|
||||
expect(unregistered).toEqual([]);
|
||||
expect(misaligned).toEqual([]);
|
||||
});
|
||||
|
||||
// Ratchet cleanup enforcement (same contract as evals-workflow-matrix's
|
||||
// burn-down test): a KNOWN_UNREGISTERED entry whose file was deleted, or
|
||||
// whose file is now named in a dep list, is stale — delete the entry so
|
||||
// the set can only shrink.
|
||||
test('KNOWN_UNREGISTERED holds only live, still-unregistered files', () => {
|
||||
const stale = [...KNOWN_UNREGISTERED].filter((repoPath) => {
|
||||
const file = repoPath.replace(/^test\//, '');
|
||||
if (!testFiles.includes(file)) return true; // file gone
|
||||
return Object.keys(allDeps).some((k) => allDeps[k].includes(repoPath)); // now registered
|
||||
});
|
||||
expect(
|
||||
stale,
|
||||
'Entry registered in a dep list or file removed — delete it from KNOWN_UNREGISTERED.',
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
// HARD invariant (C6): the paid sharded runner skips a skill-e2e shard when
|
||||
// none of the file's MAPPED test names (E2E map keys quoted in its source,
|
||||
// union E2E map keys whose dep list registers the file) are diff-selected.
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* Two invariants over paid-test timeout policy:
|
||||
*
|
||||
* 1. FIT: every tier in test/helpers/eval-budgets.ts executes inside the
|
||||
* sharded runner's wall with real overhead (bun startup + module load +
|
||||
* reporting). A budget the wall kills first is fiction — the failure
|
||||
* surfaces as a shard 'timed-out' (no bun summary, no per-test message)
|
||||
* instead of a clean test timeout. This is the structural fix for the
|
||||
* seven 1,700s-inside-a-1,500s-job literals found in the 2026-08 audit.
|
||||
*
|
||||
* 2. RATCHET: raw numeric timeout literals in paid test files only shrink.
|
||||
* New tests use the tiers; a literal is legal only with justification,
|
||||
* and the count is pinned so sprawl can't regrow.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
import { ALL_TIERS, PTY_LONG_MS } from './helpers/eval-budgets';
|
||||
import { isPaidTestFile } from './helpers/paid-test-set';
|
||||
import { DEFAULT_SHARD_TIMEOUT_MS } from '../scripts/test-paid-shards';
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
|
||||
/** Wall overhead reserve: bun startup, module load, retry bookkeeping. */
|
||||
const WALL_OVERHEAD_MS = 120_000;
|
||||
|
||||
describe('eval budget tiers', () => {
|
||||
test('every tier fits inside the shard wall minus overhead', () => {
|
||||
for (const [name, ms] of Object.entries(ALL_TIERS)) {
|
||||
expect(ms, `${name} exceeds the shard wall minus overhead`)
|
||||
.toBeLessThanOrEqual(DEFAULT_SHARD_TIMEOUT_MS - WALL_OVERHEAD_MS);
|
||||
}
|
||||
});
|
||||
|
||||
test('tiers are ordered and the ceiling is PTY_LONG', () => {
|
||||
const values = Object.values(ALL_TIERS);
|
||||
expect([...values].sort((a, b) => a - b)).toEqual(values);
|
||||
expect(Math.max(...values)).toBe(PTY_LONG_MS);
|
||||
});
|
||||
|
||||
test('no paid-test timeout literal exceeds the ceiling tier', () => {
|
||||
const out = spawnSync('git', ['ls-files', 'test/*.test.ts'], { cwd: ROOT, encoding: 'utf-8' });
|
||||
const files = out.stdout.split('\n').filter((f) => f && isPaidTestFile(f));
|
||||
expect(files.length).toBeGreaterThan(50); // scan-rot guard
|
||||
|
||||
const offenders: string[] = [];
|
||||
for (const rel of files) {
|
||||
const source = fs.readFileSync(path.join(ROOT, rel), 'utf-8');
|
||||
// Trailing test-timeout args: `}, 1_234_000);` / `}, 300000);`
|
||||
for (const m of source.matchAll(/\}\s*,\s*(\d[\d_]*)\s*(?:\/\*[^*]*\*\/\s*)?\)/g)) {
|
||||
const ms = Number(m[1].replaceAll('_', ''));
|
||||
if (ms > PTY_LONG_MS * 1.25) offenders.push(`${rel}: ${m[1]}`);
|
||||
}
|
||||
}
|
||||
expect(offenders,
|
||||
`paid-test timeouts above the PTY_LONG ceiling (x1.25 slack) are fiction ` +
|
||||
`against the ${DEFAULT_SHARD_TIMEOUT_MS / 1000}s shard wall — split the test instead:\n${offenders.join('\n')}`,
|
||||
).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,387 @@
|
||||
/**
|
||||
* The eval CLI family — scripts/eval-select.ts, eval-list.ts, eval-compare.ts,
|
||||
* eval-summary.ts — the primary interface to eval results.
|
||||
*
|
||||
* Isolation mechanisms (each verified against the source, not assumed):
|
||||
*
|
||||
* - eval-list / eval-compare / eval-summary resolve their eval dir via
|
||||
* getProjectEvalDir() (test/helpers/eval-store.ts), which probes the
|
||||
* CWD-RELATIVE `.claude/skills/gstack/bin/gstack-slug` first, then
|
||||
* `~/.claude/...` (~ = $HOME of the child). They do NOT honor
|
||||
* GSTACK_EVAL_DIR (only EvalCollector does). So the real isolation
|
||||
* mechanism is: cwd = a temp HOME containing a fake gstack-slug that
|
||||
* prints `SLUG=<fixture>`, routing every read to
|
||||
* $HOME/.gstack/projects/<fixture>/evals — fully hermetic, and it
|
||||
* exercises the primary (project-scoped) dir resolution path.
|
||||
* (test/eval-list-cli.test.ts already covers the legacy-fallback dir +
|
||||
* --limit validation; this file deliberately does not duplicate that.)
|
||||
*
|
||||
* - eval-select has NO isolation mechanism for its git diff: ROOT is
|
||||
* hardcoded to the repo containing the script (import.meta.dir/..), so
|
||||
* the CLI is smoke-tested against this repo with `--base HEAD` using
|
||||
* shape invariants that hold for any working-tree state, and the
|
||||
* "global touchfile ⇒ run everything" behavior is tested through the
|
||||
* pure, importable selectTests() the CLI is a thin wrapper over.
|
||||
*/
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { runBin } from './helpers/run-bin';
|
||||
import { selectTests, E2E_TOUCHFILES, LLM_JUDGE_TOUCHFILES, GLOBAL_TOUCHFILES } from './helpers/touchfiles';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const SCRIPT = (name: string) => path.join(ROOT, 'scripts', name);
|
||||
const SLUG = 'eval-cli-fixture';
|
||||
|
||||
let tmpHome: string;
|
||||
let evalDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-eval-family-'));
|
||||
// Fake gstack-slug at the cwd-relative probe path so getProjectEvalDir()
|
||||
// deterministically resolves the project-scoped dir under the temp HOME.
|
||||
const slugBin = path.join(tmpHome, '.claude', 'skills', 'gstack', 'bin');
|
||||
fs.mkdirSync(slugBin, { recursive: true });
|
||||
fs.writeFileSync(path.join(slugBin, 'gstack-slug'), `#!/usr/bin/env bash\necho "SLUG=${SLUG}"\n`, { mode: 0o755 });
|
||||
evalDir = path.join(tmpHome, '.gstack', 'projects', SLUG, 'evals');
|
||||
fs.mkdirSync(evalDir, { recursive: true });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tmpHome, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function runEvalCli(script: string, ...args: string[]) {
|
||||
return runBin('bun', [SCRIPT(script), ...args], {
|
||||
cwd: tmpHome,
|
||||
home: tmpHome,
|
||||
gstackHome: path.join(tmpHome, '.gstack'),
|
||||
});
|
||||
}
|
||||
|
||||
interface FixtureTest {
|
||||
name: string;
|
||||
passed: boolean;
|
||||
cost?: number;
|
||||
turns?: number;
|
||||
duration?: number;
|
||||
}
|
||||
|
||||
/** Write a run file in the collector's shapes: finalized `{version}-{branch}-{tier}-{ts}.json` or `_partial-e2e.json`. */
|
||||
function writeRun(dir: string, opts: {
|
||||
version?: string;
|
||||
branch?: string;
|
||||
tier?: 'e2e' | 'llm-judge';
|
||||
timestamp: string;
|
||||
tests: FixtureTest[];
|
||||
partial?: boolean;
|
||||
}): string {
|
||||
const version = opts.version ?? '1.0.0';
|
||||
const branch = opts.branch ?? 'featx';
|
||||
const tier = opts.tier ?? 'e2e';
|
||||
const tests = opts.tests.map(t => ({
|
||||
name: t.name,
|
||||
suite: 'fixture',
|
||||
tier,
|
||||
passed: t.passed,
|
||||
duration_ms: t.duration ?? 1000,
|
||||
cost_usd: t.cost ?? 0.5,
|
||||
turns_used: t.turns ?? 5,
|
||||
}));
|
||||
const body = {
|
||||
schema_version: 1,
|
||||
version,
|
||||
branch,
|
||||
git_sha: 'abc1234',
|
||||
timestamp: opts.timestamp,
|
||||
hostname: 'fixture-host',
|
||||
tier,
|
||||
total_tests: tests.length,
|
||||
passed: tests.filter(t => t.passed).length,
|
||||
failed: tests.filter(t => !t.passed).length,
|
||||
total_cost_usd: tests.reduce((s, t) => s + t.cost_usd, 0),
|
||||
total_duration_ms: tests.reduce((s, t) => s + t.duration_ms, 0),
|
||||
tests,
|
||||
...(opts.partial ? { _partial: true } : {}),
|
||||
};
|
||||
const dateStr = opts.timestamp.replace(/[:.]/g, '').replace('T', '-').slice(0, 15);
|
||||
const filename = opts.partial ? '_partial-e2e.json' : `${version}-${branch}-${tier}-${dateStr}.json`;
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const filepath = path.join(dir, filename);
|
||||
fs.writeFileSync(filepath, JSON.stringify(body, null, 2) + '\n');
|
||||
return filepath;
|
||||
}
|
||||
|
||||
// ── eval-select ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe('eval:select CLI (scripts/eval-select.ts)', () => {
|
||||
test('--json parses and its selection partitions the full touchfile maps', () => {
|
||||
// --base HEAD makes the committed diff empty; uncommitted/untracked files
|
||||
// in the working tree may still appear, so assert shape invariants that
|
||||
// hold for ANY tree state rather than pinning specific selections.
|
||||
const result = runBin('bun', [SCRIPT('eval-select.ts'), '--json', '--base', 'HEAD'], { cwd: ROOT });
|
||||
expect(result.status).toBe(0);
|
||||
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
expect(parsed.base).toBe('HEAD');
|
||||
|
||||
if (parsed.changed_files === 0) {
|
||||
// Pristine tree: the no-diff shape reports run-all for both tiers.
|
||||
expect(parsed.e2e).toBe('all');
|
||||
expect(parsed.llm_judge).toBe('all');
|
||||
expect(parsed.reason).toContain('all tests');
|
||||
} else {
|
||||
expect(Array.isArray(parsed.changed_files)).toBe(true);
|
||||
expect(parsed.changed_files.length).toBeGreaterThan(0);
|
||||
for (const [selection, map] of [
|
||||
[parsed.e2e, E2E_TOUCHFILES],
|
||||
[parsed.llm_judge, LLM_JUDGE_TOUCHFILES],
|
||||
] as const) {
|
||||
const total = Object.keys(map).length;
|
||||
expect(Array.isArray(selection.selected)).toBe(true);
|
||||
expect(Array.isArray(selection.skipped)).toBe(true);
|
||||
// selected + skipped always partition the map: disjoint, complete.
|
||||
expect(selection.selected.length + selection.skipped.length).toBe(total);
|
||||
const overlap = selection.selected.filter((name: string) => selection.skipped.includes(name));
|
||||
expect(overlap).toEqual([]);
|
||||
expect(typeof selection.reason).toBe('string');
|
||||
expect(selection.count).toBe(`${selection.selected.length}/${total}`);
|
||||
}
|
||||
expect(Array.isArray(parsed.e2e.removed_tests)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('human-readable mode prints the base and per-tier headers', () => {
|
||||
const result = runBin('bun', [SCRIPT('eval-select.ts'), '--base', 'HEAD'], { cwd: ROOT });
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain('Base: HEAD');
|
||||
// Either the no-diff line or the two selection headers.
|
||||
const hasNoDiff = result.stdout.includes('No changed files detected');
|
||||
if (!hasNoDiff) {
|
||||
expect(result.stdout).toContain('E2E: selected');
|
||||
expect(result.stdout).toContain('LLM-judge: selected');
|
||||
}
|
||||
});
|
||||
|
||||
test('a global-touchfile diff selects ALL tests with a global reason (pure selectTests)', () => {
|
||||
// eval-select is a thin wrapper over selectTests(); the CLI cannot be
|
||||
// pointed at a fixture repo (ROOT is hardcoded), so the run-all-on-global
|
||||
// behavior is pinned through the same imported function it calls.
|
||||
expect(GLOBAL_TOUCHFILES).toContain('test/helpers/eval-store.ts');
|
||||
const selection = selectTests(['test/helpers/eval-store.ts'], E2E_TOUCHFILES, GLOBAL_TOUCHFILES);
|
||||
expect(selection.reason).toBe('global: test/helpers/eval-store.ts');
|
||||
expect(selection.selected.sort()).toEqual(Object.keys(E2E_TOUCHFILES).sort());
|
||||
expect(selection.skipped).toEqual([]);
|
||||
});
|
||||
|
||||
test('a per-test touchfile diff selects only the dependent test', () => {
|
||||
const touchfiles = {
|
||||
'test-a': ['src/feature-a.ts', 'src/shared/**'],
|
||||
'test-b': ['src/feature-b.ts'],
|
||||
};
|
||||
const globals = ['helpers/global-runner.ts'];
|
||||
|
||||
const hitA = selectTests(['src/feature-a.ts'], touchfiles, globals);
|
||||
expect(hitA.selected).toEqual(['test-a']);
|
||||
expect(hitA.skipped).toEqual(['test-b']);
|
||||
expect(hitA.reason).toBe('diff');
|
||||
|
||||
const hitGlob = selectTests(['src/shared/deep/util.ts'], touchfiles, globals);
|
||||
expect(hitGlob.selected).toEqual(['test-a']);
|
||||
|
||||
const miss = selectTests(['docs/README.md'], touchfiles, globals);
|
||||
expect(miss.selected).toEqual([]);
|
||||
expect(miss.skipped.sort()).toEqual(['test-a', 'test-b']);
|
||||
});
|
||||
});
|
||||
|
||||
// ── eval-list ────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('eval:list CLI (scripts/eval-list.ts)', () => {
|
||||
test('empty eval dir prints the getting-started hint and exits 0', () => {
|
||||
const result = runEvalCli('eval-list.ts');
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain('No eval runs yet');
|
||||
});
|
||||
|
||||
test('lists finalized runs from the flat dir AND one level of shards/<slug>/', () => {
|
||||
writeRun(evalDir, { branch: 'flat-branch', timestamp: '2026-01-01T01:00:00Z', tests: [{ name: 't1', passed: true, cost: 1.5, turns: 7 }] });
|
||||
writeRun(path.join(evalDir, 'shards', 'shard-a'), { branch: 'shard-branch', timestamp: '2026-01-02T01:00:00Z', tests: [{ name: 't2', passed: true, cost: 0.5, turns: 3 }] });
|
||||
|
||||
const result = runEvalCli('eval-list.ts');
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain('Eval History (2 total runs)');
|
||||
expect(result.stdout).toContain('flat-branch');
|
||||
expect(result.stdout).toContain('shard-branch');
|
||||
// Sorted by timestamp descending: the shard run (newer) is listed first.
|
||||
expect(result.stdout.indexOf('shard-branch')).toBeLessThan(result.stdout.indexOf('flat-branch'));
|
||||
// Reads route to the project-scoped dir resolved via the fake gstack-slug.
|
||||
expect(result.stdout).toContain(path.join('projects', SLUG, 'evals'));
|
||||
});
|
||||
|
||||
test('--branch and --tier filter the listing', () => {
|
||||
writeRun(evalDir, { branch: 'keep-me', tier: 'e2e', timestamp: '2026-01-01T01:00:00Z', tests: [{ name: 't1', passed: true }] });
|
||||
writeRun(evalDir, { branch: 'drop-me', tier: 'llm-judge', timestamp: '2026-01-02T01:00:00Z', tests: [{ name: 't2', passed: true }] });
|
||||
|
||||
const byBranch = runEvalCli('eval-list.ts', '--branch', 'keep-me');
|
||||
expect(byBranch.status).toBe(0);
|
||||
expect(byBranch.stdout).toContain('Eval History (1 total runs)');
|
||||
expect(byBranch.stdout).toContain('keep-me');
|
||||
expect(byBranch.stdout).not.toContain('drop-me');
|
||||
|
||||
const byTier = runEvalCli('eval-list.ts', '--tier', 'llm-judge');
|
||||
expect(byTier.status).toBe(0);
|
||||
expect(byTier.stdout).toContain('drop-me');
|
||||
expect(byTier.stdout).not.toContain('keep-me');
|
||||
});
|
||||
|
||||
test('DOCUMENTS CURRENT BEHAVIOR: in-progress _partial accumulators appear in the listing', () => {
|
||||
// eval-list.ts applies NO isPartialEval filter (unlike eval-compare and
|
||||
// every baseline lookup in eval-store.ts), so the in-progress accumulator
|
||||
// is listed as if it were a run. If eval-list ever grows a partial filter,
|
||||
// update this test to assert exclusion — that would be an improvement,
|
||||
// not a regression.
|
||||
writeRun(evalDir, { branch: 'finalized-run', timestamp: '2026-01-01T01:00:00Z', tests: [{ name: 't1', passed: true }] });
|
||||
writeRun(evalDir, { branch: 'partial-sentinel', timestamp: '2026-01-03T01:00:00Z', tests: [{ name: 't1', passed: false }], partial: true });
|
||||
|
||||
const result = runEvalCli('eval-list.ts');
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain('finalized-run');
|
||||
expect(result.stdout).toContain('Eval History (2 total runs)');
|
||||
expect(result.stdout).toContain('partial-sentinel');
|
||||
});
|
||||
});
|
||||
|
||||
// ── eval-compare ─────────────────────────────────────────────────────────────
|
||||
|
||||
describe('eval:compare CLI (scripts/eval-compare.ts)', () => {
|
||||
test('empty eval dir prints the getting-started hint and exits 0', () => {
|
||||
const result = runEvalCli('eval-compare.ts');
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain('No eval runs yet');
|
||||
});
|
||||
|
||||
test('a single run is not enough to compare (exit 0 with guidance)', () => {
|
||||
writeRun(evalDir, { timestamp: '2026-01-01T01:00:00Z', tests: [{ name: 't1', passed: true }] });
|
||||
const result = runEvalCli('eval-compare.ts');
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain('Need at least 2 eval runs');
|
||||
});
|
||||
|
||||
test('no args: compares the two most recent FINALIZED runs and reports deltas; the fresher partial is never a side', () => {
|
||||
writeRun(evalDir, {
|
||||
timestamp: '2026-01-01T01:00:00Z',
|
||||
tests: [
|
||||
{ name: 't-stable', passed: true, cost: 1.0, turns: 5 },
|
||||
{ name: 't-flaky', passed: false, cost: 1.0, turns: 5 },
|
||||
{ name: 't-regressed', passed: true, cost: 1.0, turns: 5 },
|
||||
],
|
||||
});
|
||||
writeRun(evalDir, {
|
||||
timestamp: '2026-01-02T01:00:00Z',
|
||||
tests: [
|
||||
{ name: 't-stable', passed: true, cost: 1.0, turns: 5 },
|
||||
{ name: 't-flaky', passed: true, cost: 1.0, turns: 5 },
|
||||
{ name: 't-regressed', passed: false, cost: 1.0, turns: 5 },
|
||||
],
|
||||
});
|
||||
// Freshest timestamp of all — if partials leaked into selection, this
|
||||
// would be picked as the "after" run (or the baseline) and its sentinel
|
||||
// branch would show up in the header line.
|
||||
writeRun(evalDir, {
|
||||
branch: 'partial-sentinel',
|
||||
timestamp: '2026-01-03T01:00:00Z',
|
||||
tests: [{ name: 't-stable', passed: false }],
|
||||
partial: true,
|
||||
});
|
||||
|
||||
const result = runEvalCli('eval-compare.ts');
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).not.toContain('partial-sentinel');
|
||||
expect(result.stdout).toContain('1 improved');
|
||||
expect(result.stdout).toContain('1 regressed');
|
||||
expect(result.stdout).toContain('1 unchanged');
|
||||
expect(result.stdout).toContain('REGRESSION: "t-regressed" was passing, now fails.');
|
||||
expect(result.stdout).toContain('Fixed: "t-flaky" now passes.');
|
||||
});
|
||||
|
||||
test('two explicit filenames resolve relative to the eval dir and compare in the given order', () => {
|
||||
const before = writeRun(evalDir, {
|
||||
timestamp: '2026-01-01T01:00:00Z',
|
||||
tests: [{ name: 't-x', passed: true, cost: 1.0 }],
|
||||
});
|
||||
const after = writeRun(evalDir, {
|
||||
timestamp: '2026-01-02T01:00:00Z',
|
||||
tests: [{ name: 't-x', passed: false, cost: 3.0 }],
|
||||
});
|
||||
|
||||
const result = runEvalCli('eval-compare.ts', path.basename(before), path.basename(after));
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain('1 regressed');
|
||||
expect(result.stdout).toContain('REGRESSION: "t-x" was passing, now fails.');
|
||||
// Cost delta: 1.00 → 3.00 = +$2.00
|
||||
expect(result.stdout).toContain('+$2.00');
|
||||
});
|
||||
|
||||
test('a missing explicit file fails with exit 1 and names the resolved path', () => {
|
||||
writeRun(evalDir, { timestamp: '2026-01-01T01:00:00Z', tests: [{ name: 't1', passed: true }] });
|
||||
writeRun(evalDir, { timestamp: '2026-01-02T01:00:00Z', tests: [{ name: 't1', passed: true }] });
|
||||
const result = runEvalCli('eval-compare.ts', 'does-not-exist.json', 'also-missing.json');
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain('File not found:');
|
||||
expect(result.stderr).toContain('does-not-exist.json');
|
||||
});
|
||||
});
|
||||
|
||||
// ── eval-summary ─────────────────────────────────────────────────────────────
|
||||
|
||||
describe('eval:summary CLI (scripts/eval-summary.ts)', () => {
|
||||
test('empty eval dir prints the getting-started hint and exits 0', () => {
|
||||
const result = runEvalCli('eval-summary.ts');
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain('No eval runs yet');
|
||||
});
|
||||
|
||||
test('aggregates run counts, spend, and flaky tests across tiers', () => {
|
||||
writeRun(evalDir, {
|
||||
tier: 'e2e',
|
||||
branch: 'branch-one',
|
||||
timestamp: '2026-01-01T01:00:00Z',
|
||||
tests: [
|
||||
{ name: 't-flaky', passed: true, cost: 0.5, turns: 4, duration: 10_000 },
|
||||
{ name: 't-solid', passed: true, cost: 0.5, turns: 6, duration: 20_000 },
|
||||
],
|
||||
});
|
||||
writeRun(evalDir, {
|
||||
tier: 'e2e',
|
||||
branch: 'branch-one',
|
||||
timestamp: '2026-01-02T01:00:00Z',
|
||||
tests: [
|
||||
{ name: 't-flaky', passed: false, cost: 1.0, turns: 8, duration: 30_000 },
|
||||
{ name: 't-solid', passed: true, cost: 1.0, turns: 6, duration: 20_000 },
|
||||
],
|
||||
});
|
||||
writeRun(evalDir, {
|
||||
tier: 'llm-judge',
|
||||
branch: 'branch-two',
|
||||
timestamp: '2026-01-03T01:00:00Z',
|
||||
tests: [{ name: 'judge-1', passed: true, cost: 0.5 }],
|
||||
});
|
||||
|
||||
const result = runEvalCli('eval-summary.ts');
|
||||
expect(result.status).toBe(0);
|
||||
// 3 runs total: 2 e2e + 1 llm-judge.
|
||||
expect(result.stdout).toContain('3 (2 e2e, 1 llm-judge)');
|
||||
// Total spend: (0.5+0.5) + (1.0+1.0) + 0.5 = 3.50
|
||||
expect(result.stdout).toContain('$3.50');
|
||||
// t-flaky passed once and failed once → flagged flaky, keyed by tier.
|
||||
expect(result.stdout).toContain('Flaky tests (1):');
|
||||
expect(result.stdout).toContain('e2e:t-flaky');
|
||||
expect(result.stdout).not.toContain('e2e:t-solid');
|
||||
// Date range spans first → last timestamp.
|
||||
expect(result.stdout).toContain('2026-01-01 01:00');
|
||||
expect(result.stdout).toContain('2026-01-03 01:00');
|
||||
expect(result.stdout).toContain(path.join('projects', SLUG, 'evals'));
|
||||
});
|
||||
});
|
||||
@@ -48,28 +48,29 @@ const KNOWN_MATRIX_GAPS = new Set([
|
||||
'test/skill-e2e-plan-design-with-ui.test.ts',
|
||||
'test/skill-e2e-plan-devex-finding-floor.test.ts',
|
||||
'test/skill-e2e-plan-devex-plan-mode.test.ts',
|
||||
// Exposed by the 2026-08 dep-list self-registration sweep: these eight had
|
||||
// zero gate-key dep-list membership before it, so the census never saw
|
||||
// them as gate-hosting. Their gate tests run in NO CI lane today. The
|
||||
// paid-lane re-platform (test-paid-shards.ts as the CI engine) runs every
|
||||
// gate-tier file by construction and retires this whole ratchet.
|
||||
'test/skill-e2e-cso.test.ts',
|
||||
'test/skill-e2e-diagram.test.ts',
|
||||
'test/skill-e2e-learnings.test.ts',
|
||||
'test/skill-e2e-plan-tune.test.ts',
|
||||
'test/skill-e2e-plan-tune-cathedral.test.ts',
|
||||
'test/skill-e2e-review-army.test.ts',
|
||||
'test/skill-e2e-session-intelligence.test.ts',
|
||||
'test/skill-e2e-skillify.test.ts',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Matrix files whose whole-file tier guard has no matching row `tier:`
|
||||
* property (pre-existing, found 2026-08-26). Consequences today:
|
||||
* - codex-e2e / gemini-e2e declare 'periodic' → both jobs run ZERO tests and
|
||||
* report green on every PR (vestigial rows; the periodic cron lane owns
|
||||
* these suites).
|
||||
* - the two PTY plan-mode smokes declare 'gate' → the e2e-pty-plan-smoke job
|
||||
* spends ~7 min on container setup and skill registration, then bun test
|
||||
* skips every describe — hollow-green since the files adopted
|
||||
* describeE2ETier.
|
||||
* Fixing either means deliberately (re)activating paid suites on every PR —
|
||||
* tracked in the same TODOS burn-down. Fix = add `tier:` to the row (or
|
||||
* delete the vestigial row), then DELETE the entry here.
|
||||
* property. Burned down to empty 2026-08-29: the vestigial codex/gemini rows
|
||||
* were deleted (periodic-tier files, zero tests per PR) and
|
||||
* e2e-pty-plan-smoke gained its `tier: gate`. The ratchet stays so a future
|
||||
* row/file tier mismatch fails the suite instead of shipping hollow green.
|
||||
*/
|
||||
const KNOWN_TIER_UNSET = new Map([
|
||||
['test/codex-e2e.test.ts', 'periodic'],
|
||||
['test/gemini-e2e.test.ts', 'periodic'],
|
||||
['test/skill-e2e-office-hours-auto-mode.test.ts', 'gate'],
|
||||
['test/skill-e2e-plan-mode-no-op.test.ts', 'gate'],
|
||||
]);
|
||||
const KNOWN_TIER_UNSET = new Map<string, string>([]);
|
||||
|
||||
interface MatrixRow {
|
||||
name: string;
|
||||
|
||||
@@ -11,20 +11,18 @@ let gstackHome: string;
|
||||
let repoDir: string;
|
||||
|
||||
import { gitIn, findFilesBySuffix } from './helpers/scratch-repo';
|
||||
import { runBin } from './helpers/run-bin';
|
||||
|
||||
function git(args: string) {
|
||||
gitIn(repoDir, args);
|
||||
}
|
||||
|
||||
function run(args: string[], opts: { cwd?: string } = {}): { status: number; stdout: string; stderr: string } {
|
||||
const r = spawnSync(EVIDENCE, args, {
|
||||
return runBin(EVIDENCE, args, {
|
||||
cwd: opts.cwd ?? repoDir,
|
||||
env: { ...process.env, GSTACK_HOME: gstackHome },
|
||||
encoding: 'utf-8',
|
||||
timeout: 60000,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxBuffer: 16 * 1024 * 1024, // the truncation test streams 3MB through the wrapper
|
||||
});
|
||||
return { status: r.status ?? 1, stdout: r.stdout ?? '', stderr: r.stderr ?? '' };
|
||||
}
|
||||
|
||||
function ledgerFile(): string {
|
||||
|
||||
@@ -12,7 +12,8 @@ import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { spawnSync } from 'child_process';
|
||||
|
||||
import { runBin } from './helpers/run-bin';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const BIN_CONFIG = path.join(ROOT, 'bin', 'gstack-config');
|
||||
@@ -28,19 +29,9 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
function run(...args: string[]): { stdout: string; stderr: string; status: number } {
|
||||
// gstack-config precedence is `${GSTACK_HOME:-${GSTACK_STATE_DIR:-$HOME/.gstack}}`,
|
||||
// so GSTACK_HOME from the developer's parent env wins over the test's
|
||||
// GSTACK_STATE_DIR. Override both to isolate from the real ~/.gstack.
|
||||
const res = spawnSync(BIN_CONFIG, args, {
|
||||
env: { ...process.env, GSTACK_STATE_DIR: tmpHome, GSTACK_HOME: tmpHome },
|
||||
encoding: 'utf-8',
|
||||
cwd: ROOT,
|
||||
});
|
||||
return {
|
||||
stdout: (res.stdout ?? '').trim(),
|
||||
stderr: (res.stderr ?? '').trim(),
|
||||
status: res.status ?? -1,
|
||||
};
|
||||
// runBin's gstackHome sets GSTACK_HOME + GSTACK_STATE_DIR together — the
|
||||
// config-precedence isolation this file used to document by hand.
|
||||
return runBin(BIN_CONFIG, args, { gstackHome: tmpHome, cwd: ROOT, trim: true });
|
||||
}
|
||||
|
||||
describe('gstack-config explain_level', () => {
|
||||
|
||||
@@ -9,7 +9,8 @@
|
||||
* factory, opencode, openclaw, cursor, kiro).
|
||||
*
|
||||
* Tests drive gen-skill-docs as a subprocess against a temp GSTACK_HOME
|
||||
* with each detection state, then assert what landed in the generated
|
||||
* with each detection state, rendering into an isolated --out-dir (never
|
||||
* writing the working tree), then assert what landed in the rendered
|
||||
* Claude-host SKILL.md. This is end-to-end through the actual override
|
||||
* pipeline — no mocking — so it catches regressions in either the loader
|
||||
* or the suppressedResolvers filter.
|
||||
@@ -18,9 +19,9 @@
|
||||
* generation against the real repo; --host claude scopes to one host).
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { execFileSync } from 'child_process';
|
||||
import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs';
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
|
||||
@@ -49,33 +50,29 @@ function makeFixture(detectionJson: string | null): FixtureEnv {
|
||||
}
|
||||
|
||||
/**
|
||||
* Run gen-skill-docs with --respect-detection and an isolated GSTACK_HOME.
|
||||
* Returns the regenerated office-hours/SKILL.md content WITHOUT writing
|
||||
* over the committed file: we use --dry-run to keep the working tree
|
||||
* clean, then parse the output via re-reading the committed file... no,
|
||||
* that doesn't work for dry-run since dry-run doesn't write.
|
||||
*
|
||||
* Approach: generate to a temp output dir by running gen-skill-docs in a
|
||||
* temp checkout. Simpler alternative: actually regenerate, snapshot the
|
||||
* file content, then git-checkout the committed version back. We use this
|
||||
* since gen-skill-docs doesn't expose an output-path arg.
|
||||
* Run gen-skill-docs with --respect-detection and an isolated GSTACK_HOME,
|
||||
* rendering into a fresh --out-dir. The working tree is never written: the
|
||||
* generator reads its inputs (templates, resolvers) from the repo but lands
|
||||
* every output in the temp dir, which we snapshot and delete. This replaced
|
||||
* the old mutate-then-restore approach (which regenerated the committed
|
||||
* files in place and only restored the probe files, leaving every OTHER
|
||||
* generated file rewritten — a partial-restore hazard for concurrent
|
||||
* readers).
|
||||
*/
|
||||
function regenAndSnapshot(opts: {
|
||||
respectDetection: boolean;
|
||||
tmpHome: string;
|
||||
files: string[];
|
||||
}): Map<string, string> {
|
||||
// Save committed content so we can restore after snapshotting.
|
||||
const original = new Map<string, string>();
|
||||
for (const f of opts.files) {
|
||||
original.set(f, readFileSync(join(REPO_ROOT, f), 'utf-8'));
|
||||
}
|
||||
const outDir = mkdtempSync(join(tmpdir(), 'gbrain-detect-out-'));
|
||||
|
||||
const args = [
|
||||
'run',
|
||||
'scripts/gen-skill-docs.ts',
|
||||
'--host',
|
||||
'claude',
|
||||
'--out-dir',
|
||||
outDir,
|
||||
];
|
||||
if (opts.respectDetection) args.push('--respect-detection');
|
||||
|
||||
@@ -87,17 +84,14 @@ function regenAndSnapshot(opts: {
|
||||
timeout: 30_000,
|
||||
});
|
||||
|
||||
// Snapshot the regenerated content.
|
||||
// Snapshot the rendered content from the out-dir.
|
||||
const snapshot = new Map<string, string>();
|
||||
for (const f of opts.files) {
|
||||
snapshot.set(f, readFileSync(join(REPO_ROOT, f), 'utf-8'));
|
||||
snapshot.set(f, readFileSync(join(outDir, f), 'utf-8'));
|
||||
}
|
||||
return snapshot;
|
||||
} finally {
|
||||
// Always restore so the test leaves the working tree clean.
|
||||
for (const [f, content] of original) {
|
||||
writeFileSync(join(REPO_ROOT, f), content);
|
||||
}
|
||||
rmSync(outDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS } from './helpers/eval-budgets';
|
||||
import { runGeminiSkill } from './helpers/gemini-session-runner';
|
||||
import type { GeminiResult } from './helpers/gemini-session-runner';
|
||||
import { EvalCollector } from './helpers/eval-store';
|
||||
@@ -151,7 +152,7 @@ describeGemini('Gemini E2E', () => {
|
||||
// Uses a simple prompt that doesn't require skill invocation or complex navigation.
|
||||
const result = await runGeminiSkill({
|
||||
prompt: 'What is this project? Answer in one sentence based on the README.',
|
||||
timeoutMs: 90_000,
|
||||
timeoutMs: JUDGE_MS,
|
||||
cwd: testWorktree,
|
||||
});
|
||||
|
||||
@@ -163,5 +164,5 @@ describeGemini('Gemini E2E', () => {
|
||||
recordGeminiE2E('gemini-smoke', result, passed);
|
||||
|
||||
expect(result.output.length, 'Gemini should produce output').toBeGreaterThan(10);
|
||||
}, 120_000);
|
||||
}, JUDGE_MS);
|
||||
});
|
||||
|
||||
@@ -12,18 +12,26 @@
|
||||
* file's timestamp never matched the latest gen. Fixed in 43e18af4 — this
|
||||
* test pins the contract going forward.
|
||||
*
|
||||
* The test pays a small cost (~2 gen-skill-docs invocations, ~3s total) but
|
||||
* catches a class of bugs that's invisible until CI fails.
|
||||
* Isolation: each run renders into its OWN --out-dir (the working tree is
|
||||
* never written), and the two out-dirs are diffed RECURSIVELY byte-for-byte
|
||||
* — strictly stronger than the old sampled-file snapshot of an in-place
|
||||
* double regen. The only tolerated difference is the out-dir path itself:
|
||||
* --out-dir repoints section-base paths into the render, so each file is
|
||||
* normalized by replacing its own out-dir path with a placeholder before
|
||||
* comparison. Any OTHER byte difference (timestamp, random ID, iteration
|
||||
* order) still fails.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
|
||||
const REPO_ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
/** Files that gen-skill-docs writes and that must be byte-stable across runs. */
|
||||
/** Presence sanity list: key Claude-host outputs that must exist in a render
|
||||
* (guards the recursive diff against vacuously comparing two empty dirs). */
|
||||
const STABLE_OUTPUTS = [
|
||||
'SKILL.md',
|
||||
'ship/SKILL.md',
|
||||
@@ -33,10 +41,11 @@ const STABLE_OUTPUTS = [
|
||||
];
|
||||
|
||||
/**
|
||||
* Sampled outputs from EVERY non-Claude host. The full host-all run touches
|
||||
* .agents/, .cursor/, .factory/, .gbrain/, .hermes/, .kiro/, .openclaw/,
|
||||
* .opencode/, .slate/ — picking one canonical file per host catches per-host
|
||||
* non-determinism without paying the cost of snapshotting hundreds of files.
|
||||
* Presence sanity for the --host all render: one canonical file per
|
||||
* representative non-Claude host. The full host-all run touches .agents/,
|
||||
* .cursor/, .factory/, .gbrain/, .hermes/, .kiro/, .openclaw/, .opencode/,
|
||||
* .slate/ — the recursive diff covers every file; this list only proves the
|
||||
* render actually fanned out across hosts.
|
||||
*/
|
||||
const STABLE_HOST_ALL_OUTPUTS = [
|
||||
'SKILL.md',
|
||||
@@ -59,51 +68,83 @@ function runGen(extraArgs: string[] = []): { exitCode: number; stderr: string }
|
||||
};
|
||||
}
|
||||
|
||||
function snapshot(files: string[] = STABLE_OUTPUTS): Map<string, string> {
|
||||
const m = new Map<string, string>();
|
||||
for (const rel of files) {
|
||||
const full = path.join(REPO_ROOT, rel);
|
||||
if (fs.existsSync(full)) {
|
||||
m.set(rel, fs.readFileSync(full, 'utf-8'));
|
||||
}
|
||||
/** Recursively list all regular files under dir as sorted relative paths. */
|
||||
function listFiles(dir: string, prefix = ''): string[] {
|
||||
const out: string[] = [];
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
||||
const rel = prefix ? `${prefix}/${entry.name}` : entry.name;
|
||||
if (entry.isDirectory()) out.push(...listFiles(path.join(dir, entry.name), rel));
|
||||
else out.push(rel);
|
||||
}
|
||||
return m;
|
||||
return out;
|
||||
}
|
||||
|
||||
describe('gen-skill-docs idempotency', () => {
|
||||
test('two consecutive runs produce byte-identical outputs (no flapping fields)', () => {
|
||||
const firstRun = runGen();
|
||||
/**
|
||||
* Diff two render dirs recursively. Every generated output is text, so files
|
||||
* are read as utf-8 and each dir's own absolute path is normalized to
|
||||
* <OUT_DIR> (the section-base repoint is the ONLY sanctioned difference
|
||||
* between two renders of the same tree). Returns human-readable mismatches.
|
||||
*/
|
||||
function diffRenderDirs(dirA: string, dirB: string): string[] {
|
||||
const filesA = listFiles(dirA);
|
||||
const filesB = listFiles(dirB);
|
||||
const problems: string[] = [];
|
||||
const setB = new Set(filesB);
|
||||
for (const f of filesA) {
|
||||
if (!setB.has(f)) { problems.push(`${f} (only in first render)`); continue; }
|
||||
const a = fs.readFileSync(path.join(dirA, f), 'utf-8').replaceAll(dirA, '<OUT_DIR>');
|
||||
const b = fs.readFileSync(path.join(dirB, f), 'utf-8').replaceAll(dirB, '<OUT_DIR>');
|
||||
if (a !== b) problems.push(`${f} (content differs)`);
|
||||
}
|
||||
const setA = new Set(filesA);
|
||||
for (const f of filesB) {
|
||||
if (!setA.has(f)) problems.push(`${f} (only in second render)`);
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
/** Render twice into two fresh out-dirs, assert byte-identical outputs. */
|
||||
function assertDoubleRenderStable(extraArgs: string[], presenceSanity: string[], label: string): void {
|
||||
const outA = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-idem-a-'));
|
||||
const outB = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-idem-b-'));
|
||||
try {
|
||||
const firstRun = runGen([...extraArgs, '--out-dir', outA]);
|
||||
expect(firstRun.exitCode).toBe(0);
|
||||
|
||||
const after1 = snapshot();
|
||||
expect(after1.size).toBeGreaterThan(0);
|
||||
|
||||
const secondRun = runGen();
|
||||
const secondRun = runGen([...extraArgs, '--out-dir', outB]);
|
||||
expect(secondRun.exitCode).toBe(0);
|
||||
|
||||
const after2 = snapshot();
|
||||
|
||||
// Compare each stable output byte-for-byte.
|
||||
const flapping: string[] = [];
|
||||
for (const [file, before] of after1.entries()) {
|
||||
const now = after2.get(file);
|
||||
if (now !== before) flapping.push(file);
|
||||
// Non-vacuous guard: the key outputs actually rendered.
|
||||
for (const rel of presenceSanity) {
|
||||
expect({ file: rel, exists: fs.existsSync(path.join(outA, rel)) })
|
||||
.toEqual({ file: rel, exists: true });
|
||||
}
|
||||
|
||||
const flapping = diffRenderDirs(outA, outB);
|
||||
if (flapping.length > 0) {
|
||||
throw new Error(
|
||||
`${flapping.length} file(s) changed between two consecutive gen-skill-docs runs (flapping):\n` +
|
||||
`${flapping.length} file(s) differ between two consecutive ${label} gen runs (flapping):\n` +
|
||||
flapping.map(f => ` - ${f}`).join('\n') +
|
||||
`\nLikely cause: a non-deterministic field (timestamp, random ID, ` +
|
||||
`filesystem-iteration order) leaked into the generated output. CI freshness ` +
|
||||
`checks (git diff --exit-code) will fail unpredictably until this is fixed.`,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
fs.rmSync(outA, { recursive: true, force: true });
|
||||
fs.rmSync(outB, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
describe('gen-skill-docs idempotency', () => {
|
||||
test('two consecutive runs produce byte-identical outputs (no flapping fields)', () => {
|
||||
assertDoubleRenderStable([], STABLE_OUTPUTS, 'claude-host');
|
||||
}, 180_000); // ~2 min budget for two gen runs
|
||||
|
||||
test('--dry-run after a fresh gen reports zero stale files', () => {
|
||||
// Pre-condition: working tree gen must be fresh (idempotency test above ran first).
|
||||
// If a contributor introduces a non-deterministic field, this dry-run reports STALE.
|
||||
test('--dry-run against the tracked tree reports zero stale files', () => {
|
||||
// Tracked-tree freshness assertion (deliberately a READ of the committed
|
||||
// files — the out-dir renders above never touch them). If a contributor
|
||||
// edits a template without regenerating, or introduces a
|
||||
// non-deterministic field, this dry-run reports STALE.
|
||||
const result = spawnSync('bun', ['run', 'gen:skill-docs', '--dry-run'], {
|
||||
cwd: REPO_ROOT,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
@@ -115,7 +156,7 @@ describe('gen-skill-docs idempotency', () => {
|
||||
const staleLines = stdout.split('\n').filter(l => l.startsWith('STALE:'));
|
||||
if (staleLines.length > 0) {
|
||||
throw new Error(
|
||||
`--dry-run reports ${staleLines.length} stale file(s) after a fresh gen:\n` +
|
||||
`--dry-run reports ${staleLines.length} stale file(s) against the tracked tree:\n` +
|
||||
staleLines.map(l => ` ${l}`).join('\n') +
|
||||
`\nRun \`bun run gen:skill-docs\` and commit the result.`,
|
||||
);
|
||||
@@ -127,31 +168,8 @@ describe('gen-skill-docs idempotency', () => {
|
||||
// (Codex, Factory, Cursor, OpenClaw, GBrain, Slate, OpenCode, Hermes,
|
||||
// Kiro) have their own output paths and could carry their own
|
||||
// non-deterministic fields. We hit a "--host all needed for freshness
|
||||
// check" mid-/ship; this test pins the contract across every host.
|
||||
const firstRun = runGen(['--host', 'all']);
|
||||
expect(firstRun.exitCode).toBe(0);
|
||||
|
||||
const after1 = snapshot(STABLE_HOST_ALL_OUTPUTS);
|
||||
expect(after1.size).toBeGreaterThan(0);
|
||||
|
||||
const secondRun = runGen(['--host', 'all']);
|
||||
expect(secondRun.exitCode).toBe(0);
|
||||
|
||||
const after2 = snapshot(STABLE_HOST_ALL_OUTPUTS);
|
||||
|
||||
const flapping: string[] = [];
|
||||
for (const [file, before] of after1.entries()) {
|
||||
const now = after2.get(file);
|
||||
if (now !== before) flapping.push(file);
|
||||
}
|
||||
|
||||
if (flapping.length > 0) {
|
||||
throw new Error(
|
||||
`${flapping.length} file(s) changed between two consecutive --host all gen runs:\n` +
|
||||
flapping.map(f => ` - ${f}`).join('\n') +
|
||||
`\nLikely cause: a non-deterministic field leaked into a non-Claude host's ` +
|
||||
`config or resolver output. CI freshness checks for that host will flap.`,
|
||||
);
|
||||
}
|
||||
// check" mid-/ship; this test pins the contract across every host — the
|
||||
// recursive diff covers EVERY rendered file for EVERY host.
|
||||
assertDoubleRenderStable(['--host', 'all'], STABLE_HOST_ALL_OUTPUTS, '--host all');
|
||||
}, 300_000); // ~5 min budget for two host-all runs
|
||||
});
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* Importing scripts/gen-skill-docs.ts must not touch the tree.
|
||||
*
|
||||
* Before the main() guard, the generator's whole body executed at module
|
||||
* load: any `import`/`require` of it (test/gen-skill-docs.test.ts pulls
|
||||
* assertSinglePreamble; test/catalog-trim.test.ts imports helpers)
|
||||
* regenerated all 71 SKILL.md in place — the root cause of half the
|
||||
* TREE_MUTATING serial-shard entries (hazard class #2532). A regression
|
||||
* here silently re-poisons parallel shards with mid-window tree rewrites.
|
||||
*
|
||||
* The probe runs in a subprocess so a regression can't contaminate THIS
|
||||
* process, and asserts on mtimes rather than git status — the working tree
|
||||
* may legitimately carry uncommitted SKILL.md edits while this runs; what
|
||||
* must not happen is the import WRITING files.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
|
||||
describe('gen-skill-docs import purity', () => {
|
||||
test('importing the module neither writes SKILL.md nor runs main()', () => {
|
||||
const probe = `
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const ROOT = ${JSON.stringify(ROOT)};
|
||||
const targets = [
|
||||
path.join(ROOT, 'ship', 'SKILL.md'),
|
||||
path.join(ROOT, 'review', 'SKILL.md'),
|
||||
path.join(ROOT, 'gstack', 'llms.txt'),
|
||||
].filter((p) => fs.existsSync(p));
|
||||
if (targets.length === 0) throw new Error('probe rot: no generated targets found');
|
||||
const before = targets.map((p) => fs.statSync(p).mtimeMs);
|
||||
const mod = require(path.join(ROOT, 'scripts', 'gen-skill-docs.ts'));
|
||||
if (typeof mod.main !== 'function') throw new Error('main() export missing');
|
||||
const after = targets.map((p) => fs.statSync(p).mtimeMs);
|
||||
for (let i = 0; i < targets.length; i++) {
|
||||
if (before[i] !== after[i]) throw new Error('import mutated ' + targets[i]);
|
||||
}
|
||||
console.log('IMPORT_PURE');
|
||||
`;
|
||||
const out = Bun.spawnSync(['bun', '-e', probe], { cwd: ROOT });
|
||||
const stdout = out.stdout.toString();
|
||||
const stderr = out.stderr.toString();
|
||||
expect(stderr, stderr).not.toContain('import mutated');
|
||||
expect(stdout).toContain('IMPORT_PURE');
|
||||
// The import must also not have run generation output (the "GENERATED:"
|
||||
// lines main() prints) — load-time execution is the exact regression.
|
||||
expect(stdout).not.toContain('GENERATED:');
|
||||
expect(out.exitCode).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -94,4 +94,77 @@ describe('gen-skill-docs --out-dir (B2 render isolation)', () => {
|
||||
fs.rmSync(outDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// ── External-host out-dir cases ─────────────────────────────
|
||||
// The former tree-mutating tests read codex/factory artifacts from out-dir
|
||||
// renders. That is only sound if an out-dir external render is (a) clean —
|
||||
// zero tracked-tree dirt — and (b) byte-identical to what the in-place
|
||||
// render would have produced. Both halves are pinned here.
|
||||
|
||||
test('--host codex --out-dir adds no tracked dirt and is byte-identical to the in-place render', () => {
|
||||
const outDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-out-codex-'));
|
||||
const inPlaceShip = path.join(ROOT, '.agents', 'skills', 'gstack-ship', 'SKILL.md');
|
||||
// Compared before/after rather than asserting empty, so a dev's own
|
||||
// unrelated dirty files can't false-fail the suite (#2569 pattern).
|
||||
const beforePorcelain = porcelain();
|
||||
try {
|
||||
// 1) Fresh IN-PLACE codex render — the existing behavior: it writes
|
||||
// only the gitignored .agents/ tree (itself invisible to porcelain).
|
||||
const inPlace = spawnSync(
|
||||
'bun',
|
||||
['run', 'scripts/gen-skill-docs.ts', '--host', 'codex'],
|
||||
{ cwd: ROOT, encoding: 'utf-8', timeout: 120_000 },
|
||||
);
|
||||
expect(inPlace.status).toBe(0);
|
||||
expect(porcelain()).toBe(beforePorcelain);
|
||||
const inPlaceBytes = fs.readFileSync(inPlaceShip);
|
||||
|
||||
// 2) Out-dir render: zero new dirt, same bytes.
|
||||
const res = spawnSync(
|
||||
'bun',
|
||||
['run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--out-dir', outDir],
|
||||
{ cwd: ROOT, encoding: 'utf-8', timeout: 120_000 },
|
||||
);
|
||||
expect(res.status).toBe(0);
|
||||
expect(porcelain()).toBe(beforePorcelain);
|
||||
|
||||
const outShip = path.join(outDir, '.agents', 'skills', 'gstack-ship', 'SKILL.md');
|
||||
expect(fs.existsSync(outShip)).toBe(true);
|
||||
expect(fs.readFileSync(outShip).equals(inPlaceBytes)).toBe(true);
|
||||
|
||||
// Codex metadata (agents/openai.yaml) mirrors into the out-dir too.
|
||||
expect(fs.existsSync(path.join(outDir, '.agents', 'skills', 'gstack-ship', 'agents', 'openai.yaml'))).toBe(true);
|
||||
} finally {
|
||||
fs.rmSync(outDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 120_000);
|
||||
|
||||
test('--host all --out-dir renders every host tree into the out-dir; tracked tree stays clean', () => {
|
||||
const outDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-out-all-'));
|
||||
const beforePorcelain = porcelain();
|
||||
try {
|
||||
const res = spawnSync(
|
||||
'bun',
|
||||
['run', 'scripts/gen-skill-docs.ts', '--host', 'all', '--out-dir', outDir],
|
||||
{ cwd: ROOT, encoding: 'utf-8', timeout: 300_000 },
|
||||
);
|
||||
expect(res.status).toBe(0);
|
||||
// Zero new dirt in the source checkout.
|
||||
expect(porcelain()).toBe(beforePorcelain);
|
||||
|
||||
// Claude host + external hosts + openclaw docs + llms.txt all landed in the out-dir.
|
||||
for (const rel of [
|
||||
'ship/SKILL.md',
|
||||
'.agents/skills/gstack-ship/SKILL.md',
|
||||
'.factory/skills/gstack-ship/SKILL.md',
|
||||
'gstack/llms.txt',
|
||||
'openclaw/gstack-lite-CLAUDE.md',
|
||||
]) {
|
||||
expect({ file: rel, exists: fs.existsSync(path.join(outDir, rel)) })
|
||||
.toEqual({ file: rel, exists: true });
|
||||
}
|
||||
} finally {
|
||||
fs.rmSync(outDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 300_000);
|
||||
});
|
||||
|
||||
+97
-109
@@ -1,4 +1,4 @@
|
||||
import { describe, test, expect, beforeAll } from 'bun:test';
|
||||
import { describe, test, expect, afterAll } from 'bun:test';
|
||||
import { assertSinglePreamble } from '../scripts/gen-skill-docs';
|
||||
import { COMMAND_DESCRIPTIONS } from '../browse/src/commands';
|
||||
import { SNAPSHOT_FLAGS } from '../browse/src/snapshot';
|
||||
@@ -125,6 +125,32 @@ import { getHostConfig as __getHostConfig } from '../hosts/index';
|
||||
const CLAUDE_SKIPPED = new Set(__getHostConfig('claude').generation.skipSkills ?? []);
|
||||
const CLAUDE_GENERATED_SKILLS = ALL_SKILLS.filter(s => !CLAUDE_SKIPPED.has(s.dir));
|
||||
|
||||
// ─── Out-dir render isolation ────────────────────────────────
|
||||
// Every generator invocation in this file that used to regenerate the live
|
||||
// tree (the gitignored .agents/.factory/... host dirs included) now renders
|
||||
// into this module-level out-dir: ONE `--host all` render covers the claude
|
||||
// host plus every external host, and all golden-artifact reads plus the
|
||||
// per-host `--dry-run` determinism checks point here. The tracked tree is
|
||||
// only ever READ (the `generated files are fresh` dry-run deliberately
|
||||
// compares against the committed files — that is a read, not a write).
|
||||
// Out-dir renders of external hosts are byte-identical to in-place renders
|
||||
// (pinned by test/gen-skill-docs-out-dir.test.ts).
|
||||
const EXTERNAL_OUT = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-gen-docs-out-'));
|
||||
{
|
||||
const render = Bun.spawnSync(
|
||||
['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'all', '--out-dir', EXTERNAL_OUT],
|
||||
{ cwd: ROOT, stdout: 'pipe', stderr: 'pipe' },
|
||||
);
|
||||
if (render.exitCode !== 0) {
|
||||
throw new Error(
|
||||
`gen-skill-docs --host all --out-dir failed (exit ${render.exitCode}):\n${render.stderr.toString()}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
afterAll(() => {
|
||||
fs.rmSync(EXTERNAL_OUT, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('gen-skill-docs', () => {
|
||||
// Browse carve (token-reduction Phase 4): the command reference + snapshot
|
||||
// flags render into browse/sections/command-list.md now — read the
|
||||
@@ -219,8 +245,9 @@ describe('gen-skill-docs', () => {
|
||||
});
|
||||
|
||||
test('every generated Codex (.agents/skills) frontmatter parses as strict YAML', () => {
|
||||
const agentsDir = path.join(ROOT, '.agents', 'skills');
|
||||
if (!fs.existsSync(agentsDir)) return; // skip if external hosts not generated
|
||||
// Reads the module-level out-dir render (guaranteed present — the render
|
||||
// throws at module load if it fails), never the live gitignored tree.
|
||||
const agentsDir = path.join(EXTERNAL_OUT, '.agents', 'skills');
|
||||
for (const entry of fs.readdirSync(agentsDir, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
const mdPath = path.join(agentsDir, entry.name, 'SKILL.md');
|
||||
@@ -240,8 +267,7 @@ describe('gen-skill-docs', () => {
|
||||
});
|
||||
|
||||
test(`every Codex SKILL.md description stays within ${MAX_SKILL_DESCRIPTION_LENGTH} chars`, () => {
|
||||
const agentsDir = path.join(ROOT, '.agents', 'skills');
|
||||
if (!fs.existsSync(agentsDir)) return; // skip if not generated
|
||||
const agentsDir = path.join(EXTERNAL_OUT, '.agents', 'skills');
|
||||
for (const entry of fs.readdirSync(agentsDir, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
const skillMd = path.join(agentsDir, entry.name, 'SKILL.md');
|
||||
@@ -254,8 +280,7 @@ describe('gen-skill-docs', () => {
|
||||
|
||||
test('every Codex SKILL.md description stays under 900-char warning threshold', () => {
|
||||
const WARN_THRESHOLD = 900;
|
||||
const agentsDir = path.join(ROOT, '.agents', 'skills');
|
||||
if (!fs.existsSync(agentsDir)) return;
|
||||
const agentsDir = path.join(EXTERNAL_OUT, '.agents', 'skills');
|
||||
const violations: string[] = [];
|
||||
for (const entry of fs.readdirSync(agentsDir, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
@@ -283,6 +308,9 @@ describe('gen-skill-docs', () => {
|
||||
});
|
||||
|
||||
test('generated files are fresh (match --dry-run)', () => {
|
||||
// Deliberately compares against the LIVE TRACKED SKILL.md files (no
|
||||
// --out-dir): this is the freshness gate for the committed tree. Dry-run
|
||||
// writes nothing — it is a read.
|
||||
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--dry-run'], {
|
||||
cwd: ROOT,
|
||||
stdout: 'pipe',
|
||||
@@ -1373,7 +1401,7 @@ describe('DESIGN_SKETCH resolver', () => {
|
||||
|
||||
describe('CODEX_SECOND_OPINION resolver', () => {
|
||||
const content = readSkillUnion('office-hours'); // carved: Phase 5/6 prose moved to section
|
||||
const codexContent = fs.readFileSync(path.join(ROOT, '.agents', 'skills', 'gstack-office-hours', 'SKILL.md'), 'utf-8');
|
||||
const codexContent = fs.readFileSync(path.join(EXTERNAL_OUT, '.agents', 'skills', 'gstack-office-hours', 'SKILL.md'), 'utf-8');
|
||||
|
||||
test('Phase 3.5 section appears in office-hours SKILL.md', () => {
|
||||
expect(content).toContain('Phase 3.5: Cross-Model Second Opinion');
|
||||
@@ -1810,35 +1838,24 @@ describe('DESIGN_REVIEW_LITE extended with Codex', () => {
|
||||
// ─── Codex Generation Tests ─────────────────────────────────
|
||||
|
||||
describe('Codex generation (--host codex)', () => {
|
||||
const AGENTS_DIR = path.join(ROOT, '.agents', 'skills');
|
||||
// .agents/ is gitignored (v0.11.2.0) — read the module-level out-dir render
|
||||
// (--host all covers codex) instead of regenerating the live tree in place.
|
||||
const AGENTS_DIR = path.join(EXTERNAL_OUT, '.agents', 'skills');
|
||||
|
||||
// .agents/ is gitignored (v0.11.2.0) — generate on demand for tests
|
||||
Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'codex'], {
|
||||
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
|
||||
});
|
||||
|
||||
// Dynamic discovery of expected Codex skills: all templates except /codex
|
||||
// Also excludes skills where .agents/skills/{name} is a symlink back to the repo root
|
||||
// (vendored dev mode — gen-skill-docs skips these to avoid overwriting Claude SKILL.md)
|
||||
// Dynamic discovery of expected Codex skills: all templates except /codex.
|
||||
// The out-dir is a fresh mkdtemp, so the vendored-dev-mode symlink loop
|
||||
// (.agents/skills/{name} → repo root) that made the generator skip skills
|
||||
// in-place can never occur here — every template renders.
|
||||
const CODEX_SKILLS = (() => {
|
||||
const skills: Array<{ dir: string; codexName: string }> = [];
|
||||
const isSymlinkLoop = (codexName: string): boolean => {
|
||||
const agentSkillDir = path.join(ROOT, '.agents', 'skills', codexName);
|
||||
try {
|
||||
return fs.realpathSync(agentSkillDir) === fs.realpathSync(ROOT);
|
||||
} catch { return false; }
|
||||
};
|
||||
if (fs.existsSync(path.join(ROOT, 'SKILL.md.tmpl'))) {
|
||||
if (!isSymlinkLoop('gstack')) {
|
||||
skills.push({ dir: '.', codexName: 'gstack' });
|
||||
}
|
||||
skills.push({ dir: '.', codexName: 'gstack' });
|
||||
}
|
||||
for (const entry of fs.readdirSync(ROOT, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory() || entry.name.startsWith('.') || entry.name === 'node_modules') continue;
|
||||
if (entry.name === 'codex') continue; // /codex is excluded from Codex output
|
||||
if (!fs.existsSync(path.join(ROOT, entry.name, 'SKILL.md.tmpl'))) continue;
|
||||
const codexName = entry.name.startsWith('gstack-') ? entry.name : `gstack-${entry.name}`;
|
||||
if (isSymlinkLoop(codexName)) continue;
|
||||
skills.push({ dir: entry.name, codexName });
|
||||
}
|
||||
return skills;
|
||||
@@ -1967,7 +1984,9 @@ describe('Codex generation (--host codex)', () => {
|
||||
});
|
||||
|
||||
test('--host codex --dry-run freshness', () => {
|
||||
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--dry-run'], {
|
||||
// Dry-run against the out-dir render: determinism/idempotency check
|
||||
// (regenerating produces the same bytes the module-level render did).
|
||||
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--dry-run', '--out-dir', EXTERNAL_OUT], {
|
||||
cwd: ROOT,
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
@@ -1982,12 +2001,12 @@ describe('Codex generation (--host codex)', () => {
|
||||
});
|
||||
|
||||
test('--host agents alias produces same output as --host codex', () => {
|
||||
const codexResult = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--dry-run'], {
|
||||
const codexResult = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--dry-run', '--out-dir', EXTERNAL_OUT], {
|
||||
cwd: ROOT,
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
});
|
||||
const agentsResult = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'agents', '--dry-run'], {
|
||||
const agentsResult = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'agents', '--dry-run', '--out-dir', EXTERNAL_OUT], {
|
||||
cwd: ROOT,
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
@@ -2195,63 +2214,52 @@ describe('Codex generation (--host codex)', () => {
|
||||
// ─── Explicit --model override wins over the host default ────
|
||||
// Without --model the codex host renders its defaultModel (gpt) — pinned by
|
||||
// the golden test. This pins the OTHER direction through the real CLI:
|
||||
// `./setup --host codex --model <id>` depends on it. Runs last in this
|
||||
// describe and restores the host-default render before finishing.
|
||||
// `./setup --host codex --model <id>` depends on it. The override renders
|
||||
// into its OWN out-dir, so no restore pass is needed — the host-default
|
||||
// render (EXTERNAL_OUT) is untouched and asserted directly.
|
||||
test('explicit --model overrides the codex host default', () => {
|
||||
const overrideOut = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-model-override-'));
|
||||
try {
|
||||
const override = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--model', 'claude'], {
|
||||
const override = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'codex', '--model', 'claude', '--out-dir', overrideOut], {
|
||||
cwd: ROOT,
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
});
|
||||
expect(override.exitCode).toBe(0);
|
||||
const content = fs.readFileSync(path.join(AGENTS_DIR, 'gstack-ship', 'SKILL.md'), 'utf-8');
|
||||
const content = fs.readFileSync(path.join(overrideOut, '.agents', 'skills', 'gstack-ship', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('Model-Specific Behavioral Patch (claude)');
|
||||
// The overlay now travels as --model into gstack-skill-start, which
|
||||
// echoes MODEL_OVERLAY at runtime.
|
||||
expect(content).toContain('--model "claude"');
|
||||
} finally {
|
||||
// Restore the host-default render — later tests and the host-config
|
||||
// golden read this tree.
|
||||
const restore = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'codex'], {
|
||||
cwd: ROOT,
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
});
|
||||
expect(restore.exitCode).toBe(0);
|
||||
fs.rmSync(overrideOut, { recursive: true, force: true });
|
||||
}
|
||||
const restored = fs.readFileSync(path.join(AGENTS_DIR, 'gstack-ship', 'SKILL.md'), 'utf-8');
|
||||
expect(restored).toContain('Model-Specific Behavioral Patch (gpt)');
|
||||
expect(restored).toContain('--model "gpt"');
|
||||
// Host-default direction: the untouched EXTERNAL_OUT render carries gpt.
|
||||
const hostDefault = fs.readFileSync(path.join(AGENTS_DIR, 'gstack-ship', 'SKILL.md'), 'utf-8');
|
||||
expect(hostDefault).toContain('Model-Specific Behavioral Patch (gpt)');
|
||||
expect(hostDefault).toContain('--model "gpt"');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Factory generation tests ────────────────────────────────
|
||||
|
||||
describe('Factory generation (--host factory)', () => {
|
||||
const FACTORY_DIR = path.join(ROOT, '.factory', 'skills');
|
||||
|
||||
// Generate Factory output for tests
|
||||
Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'factory'], {
|
||||
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
|
||||
});
|
||||
// .factory/ is gitignored — read the module-level out-dir render
|
||||
// (--host all covers factory) instead of regenerating in place.
|
||||
const FACTORY_DIR = path.join(EXTERNAL_OUT, '.factory', 'skills');
|
||||
|
||||
// Fresh out-dir → the vendored-dev-mode symlink loop can never occur, so
|
||||
// every template renders (see the Codex discovery note above).
|
||||
const FACTORY_SKILLS = (() => {
|
||||
const skills: Array<{ dir: string; factoryName: string }> = [];
|
||||
const isSymlinkLoop = (name: string): boolean => {
|
||||
const factorySkillDir = path.join(ROOT, '.factory', 'skills', name);
|
||||
try { return fs.realpathSync(factorySkillDir) === fs.realpathSync(ROOT); }
|
||||
catch { return false; }
|
||||
};
|
||||
if (fs.existsSync(path.join(ROOT, 'SKILL.md.tmpl'))) {
|
||||
if (!isSymlinkLoop('gstack')) skills.push({ dir: '.', factoryName: 'gstack' });
|
||||
skills.push({ dir: '.', factoryName: 'gstack' });
|
||||
}
|
||||
for (const entry of fs.readdirSync(ROOT, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory() || entry.name.startsWith('.') || entry.name === 'node_modules') continue;
|
||||
if (entry.name === 'codex') continue;
|
||||
if (!fs.existsSync(path.join(ROOT, entry.name, 'SKILL.md.tmpl'))) continue;
|
||||
const factoryName = entry.name.startsWith('gstack-') ? entry.name : `gstack-${entry.name}`;
|
||||
if (isSymlinkLoop(factoryName)) continue;
|
||||
skills.push({ dir: entry.name, factoryName });
|
||||
}
|
||||
return skills;
|
||||
@@ -2333,10 +2341,10 @@ describe('Factory generation (--host factory)', () => {
|
||||
});
|
||||
|
||||
test('--host droid alias works', () => {
|
||||
const factoryResult = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'factory', '--dry-run'], {
|
||||
const factoryResult = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'factory', '--dry-run', '--out-dir', EXTERNAL_OUT], {
|
||||
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
|
||||
});
|
||||
const droidResult = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'droid', '--dry-run'], {
|
||||
const droidResult = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'droid', '--dry-run', '--out-dir', EXTERNAL_OUT], {
|
||||
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
|
||||
});
|
||||
expect(factoryResult.exitCode).toBe(0);
|
||||
@@ -2345,7 +2353,7 @@ describe('Factory generation (--host factory)', () => {
|
||||
});
|
||||
|
||||
test('--host factory --dry-run freshness', () => {
|
||||
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'factory', '--dry-run'], {
|
||||
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'factory', '--dry-run', '--out-dir', EXTERNAL_OUT], {
|
||||
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
|
||||
});
|
||||
expect(result.exitCode).toBe(0);
|
||||
@@ -2369,33 +2377,19 @@ describe('Factory generation (--host factory)', () => {
|
||||
import { ALL_HOST_CONFIGS, getExternalHosts } from '../hosts/index';
|
||||
|
||||
describe('Parameterized host smoke tests', () => {
|
||||
// Regenerate every external host up front so the per-host `--dry-run` freshness
|
||||
// checks are deterministic. These host dirs (.agents/.factory/.cursor/...) are
|
||||
// gitignored regenerated artifacts, so the freshness check is really an
|
||||
// idempotency/determinism check — it still catches non-deterministic gen, but no
|
||||
// longer flakes on stale-on-disk state left by a missing `gen --host all` prestep
|
||||
// (the canonical `bun test` does not run one). The tracked-claude freshness test
|
||||
// Every external host was rendered up front by the module-level
|
||||
// `--host all --out-dir EXTERNAL_OUT` render, so the per-host `--dry-run`
|
||||
// freshness checks are deterministic: they compare a regeneration against
|
||||
// that render — an idempotency/determinism check that catches
|
||||
// non-deterministic gen without ever writing (or depending on) the live
|
||||
// gitignored host dirs. The tracked-claude freshness test
|
||||
// (`generated files are fresh`) runs earlier and is unaffected.
|
||||
beforeAll(() => {
|
||||
for (const h of getExternalHosts()) {
|
||||
Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', h.name], {
|
||||
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
for (const hostConfig of getExternalHosts()) {
|
||||
describe(`${hostConfig.displayName} (--host ${hostConfig.name})`, () => {
|
||||
const hostDir = path.join(ROOT, hostConfig.hostSubdir, 'skills');
|
||||
const hostDir = path.join(EXTERNAL_OUT, hostConfig.hostSubdir, 'skills');
|
||||
|
||||
test('generates output that exists on disk', () => {
|
||||
// Generated dir should exist (created by earlier bun run gen:skill-docs --host all)
|
||||
if (!fs.existsSync(hostDir)) {
|
||||
// Generate if not already done
|
||||
Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', hostConfig.name], {
|
||||
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
|
||||
});
|
||||
}
|
||||
// The module-level --host all render must have produced this host's tree.
|
||||
expect(fs.existsSync(hostDir)).toBe(true);
|
||||
const skills = fs.readdirSync(hostDir).filter(d =>
|
||||
fs.existsSync(path.join(hostDir, d, 'SKILL.md'))
|
||||
@@ -2437,7 +2431,7 @@ describe('Parameterized host smoke tests', () => {
|
||||
|
||||
test('--dry-run freshness check passes', () => {
|
||||
const result = Bun.spawnSync(
|
||||
['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', hostConfig.name, '--dry-run'],
|
||||
['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', hostConfig.name, '--dry-run', '--out-dir', EXTERNAL_OUT],
|
||||
{ cwd: ROOT, stdout: 'pipe', stderr: 'pipe' }
|
||||
);
|
||||
expect(result.exitCode).toBe(0);
|
||||
@@ -2457,18 +2451,12 @@ describe('Parameterized host smoke tests', () => {
|
||||
// ─── --host all tests ────────────────────────────────────────
|
||||
|
||||
describe('--host all', () => {
|
||||
// Same determinism guard as the parameterized block: make external hosts fresh on
|
||||
// disk so `--host all --dry-run` reports FRESH regardless of prior state.
|
||||
beforeAll(() => {
|
||||
for (const h of getExternalHosts()) {
|
||||
Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', h.name], {
|
||||
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Same determinism guard as the parameterized block: the module-level
|
||||
// `--host all --out-dir EXTERNAL_OUT` render is the comparison baseline, so
|
||||
// this dry-run reports FRESH regardless of live-tree state — and proves the
|
||||
// claude host plus every external host regenerate deterministically.
|
||||
test('--host all generates for all registered hosts', () => {
|
||||
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'all', '--dry-run'], {
|
||||
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'all', '--dry-run', '--out-dir', EXTERNAL_OUT], {
|
||||
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
|
||||
});
|
||||
expect(result.exitCode).toBe(0);
|
||||
@@ -2636,8 +2624,8 @@ describe('setup script validation', () => {
|
||||
|
||||
// T2: Dynamic $GSTACK_ROOT paths in generated Codex preambles
|
||||
test('generated Codex preambles use dynamic GSTACK_ROOT paths', () => {
|
||||
const codexSkillDir = path.join(ROOT, '.agents', 'skills', 'gstack-ship');
|
||||
if (!fs.existsSync(codexSkillDir)) return; // skip if .agents/ not generated
|
||||
// Read the module-level out-dir render (always present).
|
||||
const codexSkillDir = path.join(EXTERNAL_OUT, '.agents', 'skills', 'gstack-ship');
|
||||
const content = fs.readFileSync(path.join(codexSkillDir, 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('GSTACK_ROOT=');
|
||||
expect(content).toContain('$GSTACK_BIN/');
|
||||
@@ -3332,7 +3320,10 @@ describe('gen-skill-docs prefix warning (#620/#578)', () => {
|
||||
fs.mkdirSync(fakeGstack, { recursive: true });
|
||||
fs.writeFileSync(path.join(fakeGstack, 'config.yaml'), 'skill_prefix: true\n');
|
||||
|
||||
const output = execSync('bun run scripts/gen-skill-docs.ts', {
|
||||
// Render into an out-dir under the fixture (the warning fires on any
|
||||
// non-dry-run generation) so the live tree is never rewritten.
|
||||
const outDir = path.join(tmpDir, 'out');
|
||||
const output = execSync(`bun run scripts/gen-skill-docs.ts --out-dir "${outDir}"`, {
|
||||
cwd: ROOT,
|
||||
env: { ...process.env, HOME: fakeHome },
|
||||
encoding: 'utf-8',
|
||||
@@ -3353,7 +3344,8 @@ describe('gen-skill-docs prefix warning (#620/#578)', () => {
|
||||
fs.mkdirSync(fakeGstack, { recursive: true });
|
||||
fs.writeFileSync(path.join(fakeGstack, 'config.yaml'), 'skill_prefix: false\n');
|
||||
|
||||
const output = execSync('bun run scripts/gen-skill-docs.ts', {
|
||||
const outDir = path.join(tmpDir, 'out');
|
||||
const output = execSync(`bun run scripts/gen-skill-docs.ts --out-dir "${outDir}"`, {
|
||||
cwd: ROOT,
|
||||
env: { ...process.env, HOME: fakeHome },
|
||||
encoding: 'utf-8',
|
||||
@@ -3469,14 +3461,16 @@ describe('plan-mode-info resolver (handshake-replacement)', () => {
|
||||
expect(checked).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('vestigial handshake is absent from non-Claude host outputs when present on disk', () => {
|
||||
test('vestigial handshake is absent from non-Claude host outputs', () => {
|
||||
// Non-Claude hosts render to hostSubdirs (.agents/, .openclaw/, etc). The
|
||||
// plan-mode-info resolver has no host-scoping — all hosts get the new
|
||||
// section, none get the old handshake. Scan all candidate host dirs.
|
||||
// section, none get the old handshake. Scan every candidate host tree in
|
||||
// the module-level out-dir render (--host all), which is always present —
|
||||
// so the check can no longer silently degrade to a console warning.
|
||||
const hostDirs = ['.agents', '.openclaw', '.opencode', '.factory', '.hermes', '.kiro', '.cursor', '.slate'];
|
||||
let checked = 0;
|
||||
for (const host of hostDirs) {
|
||||
const skillsRoot = path.join(ROOT, host, 'skills');
|
||||
const skillsRoot = path.join(EXTERNAL_OUT, host, 'skills');
|
||||
if (!fs.existsSync(skillsRoot)) continue;
|
||||
const entries = fs.readdirSync(skillsRoot, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
@@ -3488,13 +3482,7 @@ describe('plan-mode-info resolver (handshake-replacement)', () => {
|
||||
checked++;
|
||||
}
|
||||
}
|
||||
if (checked === 0) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(
|
||||
'plan-mode-info: no non-Claude host outputs found for cross-host absence check — ' +
|
||||
'run `bun run gen:skill-docs --host all` to populate',
|
||||
);
|
||||
}
|
||||
expect(checked).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test.each(REVIEW_SKILLS)(
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
/**
|
||||
* No module-scope GSTACK_HOME assignment in any test file.
|
||||
*
|
||||
* Shard processes evaluate many test-file modules in one bun process, and a
|
||||
* module can be loaded before its tests run — so a module-scope
|
||||
* `process.env.GSTACK_HOME = ...` leaks into every sibling file in the
|
||||
* shard. The damage was real before the 2026-08 sweep: relink.test.ts:28
|
||||
* documents a "fresh install" test seeing a neighbor's skill_prefix, and
|
||||
* cdp-e2e once baked a sibling's temp dir into artifacts that outlived it
|
||||
* (dangling symlinks into a deleted render dir).
|
||||
*
|
||||
* The pattern is: save the original, assign in beforeAll, restore in
|
||||
* afterAll — confining the value to the file's execution window. See
|
||||
* browse/test/cdp-e2e.test.ts for the reference shape.
|
||||
*
|
||||
* Heuristic: repo test files write module-scope statements unindented, so a
|
||||
* column-0 assignment is module scope; indented assignments (inside hooks,
|
||||
* tests, or helpers) are fine.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
|
||||
function trackedTestFiles(): string[] {
|
||||
const out = spawnSync('git', ['ls-files', '*.test.ts'], {
|
||||
cwd: ROOT, encoding: 'utf-8',
|
||||
});
|
||||
if (out.status !== 0) throw new Error(`git ls-files failed: ${out.stderr}`);
|
||||
return out.stdout.split('\n').filter(Boolean);
|
||||
}
|
||||
|
||||
describe('GSTACK_HOME module-scope tripwire', () => {
|
||||
test('no test file assigns process.env.GSTACK_HOME at module scope', () => {
|
||||
const files = trackedTestFiles();
|
||||
expect(files.length).toBeGreaterThan(100); // scan-rot guard
|
||||
|
||||
const offenders: string[] = [];
|
||||
for (const rel of files) {
|
||||
const lines = fs.readFileSync(path.join(ROOT, rel), 'utf-8').split('\n');
|
||||
lines.forEach((line, i) => {
|
||||
if (/^(?:process\.env\.GSTACK_HOME|process\.env\.GSTACK_STATE_ROOT)\s*=[^=]/.test(line)) {
|
||||
offenders.push(`${rel}:${i + 1} — ${line.trim()}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
expect(offenders,
|
||||
`module-scope env assignment leaks across shard siblings — move into beforeAll + restore in afterAll:\n${offenders.join('\n')}`,
|
||||
).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -8,16 +8,28 @@
|
||||
* timestamp + scope + reason + CI provenance.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeEach } from 'bun:test';
|
||||
import { describe, test, expect, beforeAll, beforeEach, afterAll } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { logBudgetOverride } from './budget-override';
|
||||
|
||||
const TMP_HOME = fs.mkdtempSync(path.join(os.tmpdir(), 'budget-override-test-'));
|
||||
process.env.GSTACK_HOME = TMP_HOME;
|
||||
const AUDIT_PATH = path.join(TMP_HOME, 'analytics', 'spend-overrides.jsonl');
|
||||
|
||||
// GSTACK_HOME is scoped to this file's execution window (beforeAll/afterAll),
|
||||
// never set at module load: bun evaluates sibling modules before running
|
||||
// their tests, so a module-scope assignment leaks into every other file in
|
||||
// the shard process (pinned by test/gstack-home-module-scope.test.ts).
|
||||
const ORIGINAL_GSTACK_HOME = process.env.GSTACK_HOME;
|
||||
beforeAll(() => {
|
||||
process.env.GSTACK_HOME = TMP_HOME;
|
||||
});
|
||||
afterAll(() => {
|
||||
if (ORIGINAL_GSTACK_HOME === undefined) delete process.env.GSTACK_HOME;
|
||||
else process.env.GSTACK_HOME = ORIGINAL_GSTACK_HOME;
|
||||
});
|
||||
|
||||
describe('logBudgetOverride', () => {
|
||||
beforeEach(() => {
|
||||
// Start each test with a clean audit file
|
||||
|
||||
@@ -61,7 +61,55 @@ export function computeDiffSelection(
|
||||
return selection.selected;
|
||||
}
|
||||
|
||||
export let selectedTests: string[] | null = computeDiffSelection(E2E_TOUCHFILES, 'E2E'); // null = run all
|
||||
/**
|
||||
* Parse the sharded paid runner's precomputed selection (EVALS_SELECTION_JSON,
|
||||
* written by serializePaidDiffSelection in scripts/test-paid-shards.ts).
|
||||
* Returns { selected: null } for run-all. THROWS on any parse/shape failure —
|
||||
* resolveModuleSelection turns that into a fail-open local recompute.
|
||||
*/
|
||||
export function parseEvalsSelectionJson(raw: string): { selected: string[] | null; reason: string } {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error('not an object');
|
||||
const { selected, reason } = parsed as { selected?: unknown; reason?: unknown };
|
||||
if (selected !== null
|
||||
&& !(Array.isArray(selected) && selected.every((s) => typeof s === 'string'))) {
|
||||
throw new Error('selected must be null or string[]');
|
||||
}
|
||||
return {
|
||||
selected: selected as string[] | null,
|
||||
reason: typeof reason === 'string' ? reason : 'parent selection',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the module-load E2E selection: prefer the parent shard runner's
|
||||
* EVALS_SELECTION_JSON — skipping this module's own git walk and, when
|
||||
* touchfiles-data.ts is in the diff, the per-child bun subprocess that
|
||||
* evaluates the old data file (test-selection.ts map-diff path, one per
|
||||
* shard). On ANY parse/shape failure, fall back to computing locally
|
||||
* (fail-open preserved) with one stderr warning.
|
||||
*/
|
||||
export function resolveModuleSelection(
|
||||
raw: string | undefined,
|
||||
compute: () => string[] | null,
|
||||
stderrWrite: (text: string) => void = (text) => process.stderr.write(text),
|
||||
): string[] | null {
|
||||
if (raw) {
|
||||
try {
|
||||
const { selected, reason } = parseEvalsSelectionJson(raw);
|
||||
stderrWrite(`\nE2E selection (parent-propagated: ${reason}): ${selected === null ? 'all' : selected.length} tests\n`);
|
||||
return selected;
|
||||
} catch (err) {
|
||||
stderrWrite(`WARNING: malformed EVALS_SELECTION_JSON (${err instanceof Error ? err.message : String(err)}) — falling back to local selection\n`);
|
||||
}
|
||||
}
|
||||
return compute();
|
||||
}
|
||||
|
||||
export let selectedTests: string[] | null = resolveModuleSelection(
|
||||
evalsEnabled ? process.env.EVALS_SELECTION_JSON : undefined,
|
||||
() => computeDiffSelection(E2E_TOUCHFILES, 'E2E'),
|
||||
); // null = run all
|
||||
|
||||
// EVALS_TIER: filter tests by tier after diff-based selection.
|
||||
// 'gate' = gate tests only (CI default — blocks merge)
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
/**
|
||||
* Timeout policy for paid tests — five tiers instead of hand-tuned sprawl.
|
||||
*
|
||||
* Before this module the paid suite carried 46×300s, 46×120s, 44×360s,
|
||||
* 44×180s, 27×240s, 19×150s, 13×420s, 12×600s, 7×700s… hand-ratcheted
|
||||
* per test, several inflated to paper over the old 40-way in-shard
|
||||
* concurrency (session startup queued behind 39 siblings and ate the
|
||||
* budget before turn one — dead with the sharded runner's 1-file-per-shard
|
||||
* model). Pick the tier that matches the test's SHAPE; escape-hatch raw
|
||||
* literals stay legal with a justification comment (count-ratcheted by
|
||||
* test/eval-budgets-policy.test.ts).
|
||||
*
|
||||
* Every tier must fit inside the lane walls — pinned by the fit test in
|
||||
* test/eval-budgets-policy.test.ts against the sharded runner's
|
||||
* DEFAULT_SHARD_TIMEOUT_MS. Budget above the wall is fiction, not headroom.
|
||||
*/
|
||||
|
||||
/** LLM-judge call over an existing capture (no agent session). */
|
||||
export const JUDGE_MS = 120_000;
|
||||
|
||||
/** One `claude -p` / SDK capture, bounded turns. */
|
||||
export const CAPTURE_MS = 300_000;
|
||||
|
||||
/** Multi-capture or long multi-turn `claude -p` flows. */
|
||||
export const CAPTURE_LONG_MS = 600_000;
|
||||
|
||||
/** Interactive real-PTY flow (spawn + skill + a few interactions). */
|
||||
export const PTY_MS = 900_000;
|
||||
|
||||
/**
|
||||
* Chained/judged PTY observation — the ceiling tier. 1200s leaves the
|
||||
* 1800s shard wall real overhead; anything that genuinely needs more
|
||||
* should be SPLIT, not budgeted past the wall.
|
||||
*/
|
||||
export const PTY_LONG_MS = 1_200_000;
|
||||
|
||||
export const ALL_TIERS = {
|
||||
JUDGE_MS,
|
||||
CAPTURE_MS,
|
||||
CAPTURE_LONG_MS,
|
||||
PTY_MS,
|
||||
PTY_LONG_MS,
|
||||
} as const;
|
||||
+30
-12
@@ -11,6 +11,8 @@
|
||||
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
|
||||
import { resolveEvalModel } from '../../lib/eval-model';
|
||||
|
||||
export interface JudgeScore {
|
||||
clarity: number; // 1-5
|
||||
completeness: number; // 1-5
|
||||
@@ -52,9 +54,10 @@ export interface RecommendationScore {
|
||||
|
||||
/**
|
||||
* Call an Anthropic model with a prompt, extract JSON response.
|
||||
* Retries once on 429 rate limit errors. Defaults to Sonnet 4.6 for
|
||||
* existing callers; pass a model id (e.g. claude-haiku-4-5-20251001)
|
||||
* for cheaper bounded judgments like judgeRecommendation.
|
||||
* Jittered exponential backoff over three 429 retries. Model resolves via
|
||||
* lib/eval-model's `judge` kind (Sonnet default); pass a model id
|
||||
* (e.g. claude-haiku-4-5-20251001) for cheaper bounded judgments like
|
||||
* judgeRecommendation.
|
||||
*/
|
||||
// Default judge model: Sonnet. D1a tried Haiku 4.5 here and the first live
|
||||
// run regressed the doc-rubric family — a controlled A/B on the identical
|
||||
@@ -68,27 +71,42 @@ export interface RecommendationScore {
|
||||
// distill — see lib/eval-model.ts).
|
||||
export async function callJudge<T>(
|
||||
prompt: string,
|
||||
model: string = process.env.GSTACK_EVAL_MODEL_JUDGE || 'claude-sonnet-4-6',
|
||||
model?: string,
|
||||
opts?: { temperature?: number; max_tokens?: number },
|
||||
): Promise<T> {
|
||||
// Routed through the documented single resolution point: explicit arg >
|
||||
// GSTACK_EVAL_MODEL_JUDGE > GSTACK_EVAL_MODEL > sonnet default. The old
|
||||
// inline `GSTACK_EVAL_MODEL_JUDGE || sonnet` silently ignored the global
|
||||
// GSTACK_EVAL_MODEL override that every other eval call site honors.
|
||||
// opts (temperature/max_tokens) exist for bounded judgments like armJudge;
|
||||
// defaults preserve prior behavior.
|
||||
const resolvedModel = resolveEvalModel('judge', model);
|
||||
const client = new Anthropic();
|
||||
|
||||
const makeRequest = () => client.messages.create({
|
||||
model,
|
||||
model: resolvedModel,
|
||||
max_tokens: opts?.max_tokens ?? 1024,
|
||||
...(opts?.temperature !== undefined ? { temperature: opts.temperature } : {}),
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
});
|
||||
|
||||
// 429s under CI concurrency: jittered exponential backoff over 3 retries
|
||||
// (~1s/4s/16s + jitter), honoring the server's retry-after when present.
|
||||
// The old single fixed 1s retry lost races reliably at 40-way concurrency.
|
||||
let response;
|
||||
try {
|
||||
response = await makeRequest();
|
||||
} catch (err: any) {
|
||||
if (err.status === 429) {
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
let attempt = 0;
|
||||
for (;;) {
|
||||
try {
|
||||
response = await makeRequest();
|
||||
} else {
|
||||
throw err;
|
||||
break;
|
||||
} catch (err: any) {
|
||||
if (err?.status !== 429 || attempt >= 3) throw err;
|
||||
const retryAfterSecs = Number(err?.headers?.['retry-after']);
|
||||
const baseMs = Number.isFinite(retryAfterSecs) && retryAfterSecs > 0
|
||||
? retryAfterSecs * 1000
|
||||
: 1000 * 4 ** attempt;
|
||||
await new Promise((r) => setTimeout(r, baseMs + Math.random() * 500));
|
||||
attempt += 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -11,12 +11,20 @@ import { matchGlob } from './touchfiles';
|
||||
|
||||
/** The exact globs package.json's `test:gate` passes to `bun test`. */
|
||||
export const PAID_TEST_GLOBS = [
|
||||
'test/skill-llm-eval.test.ts',
|
||||
// skill-llm-eval* (not just the base file): skill-llm-eval-spec.test.ts
|
||||
// fell outside the exact glob and could never run in any lane.
|
||||
'test/skill-llm-eval*.test.ts',
|
||||
'test/skill-e2e-*.test.ts',
|
||||
'test/skill-routing-e2e.test.ts',
|
||||
'test/codex-e2e.test.ts',
|
||||
'test/codex-e2e-sol-scope.test.ts',
|
||||
// codex-e2e* (was two exact names): codex-e2e-plan-format.test.ts and
|
||||
// codex-e2e-recommendation-substance.test.ts were API-spending orphans —
|
||||
// outside these globs they self-skipped in the free suite AND never
|
||||
// entered the paid census. The same bug class as the deleted pre-split
|
||||
// monolith (see test/paid-shards.test.ts's regression pin).
|
||||
'test/codex-e2e*.test.ts',
|
||||
'test/gemini-e2e.test.ts',
|
||||
'test/llm-judge-recommendation.test.ts',
|
||||
'test/carve-section-loading.test.ts',
|
||||
] as const;
|
||||
|
||||
/** True when a repo-relative path (either slash style) is a paid test file. */
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
/**
|
||||
* Periodic-lane exclusions — LITERALS ONLY (own file, deliberately NOT in
|
||||
* touchfiles-data.ts: that file is evaluated standalone by map-diff against
|
||||
* old git versions, and its contract must not grow unrelated exports).
|
||||
*
|
||||
* The weekly periodic CI lane runs EVERY periodic-tier file (EVALS_ALL=1) so
|
||||
* tests can't rot invisibly — the coverage contract. A file lands here only
|
||||
* when running it weekly is KNOWN waste (documented-red or requires manual
|
||||
* hardware), and every entry must carry a tracking pointer with a re-entry
|
||||
* condition, so an exclusion is a decision with an owner, not a place tests
|
||||
* go to die. Pinned by test/periodic-exclude-policy.test.ts: entries must
|
||||
* name real files and carry non-empty reason + tracking.
|
||||
*
|
||||
* Removing an entry re-activates the file on the next weekly run — that IS
|
||||
* the re-entry mechanism.
|
||||
*/
|
||||
export const PERIODIC_CI_EXCLUDE: Record<string, { reason: string; tracking: string }> = {
|
||||
'test/skill-e2e-ship-idempotency.test.ts': {
|
||||
reason:
|
||||
'documented-red: the PTY child sits at the Claude Code welcome screen for the full budget '
|
||||
+ '(readiness/typing race vs CLI 2.1.x); never green since it was born in v1.63',
|
||||
tracking: 'TODOS.md "periodic tier — three documented-red tests need structural repair" (1 of 3 resolved: sidebar trio already deleted)',
|
||||
},
|
||||
'test/skill-e2e-brain-privacy-gate.test.ts': {
|
||||
reason:
|
||||
'documented-red: the artifacts-sync stop-gate preconditions do not survive the hermetic env '
|
||||
+ 'even with per-test HOME/GSTACK_HOME injection; never green anywhere',
|
||||
tracking: 'TODOS.md "periodic tier — three documented-red tests need structural repair"',
|
||||
},
|
||||
'test/skill-e2e-ios.test.ts': {
|
||||
reason: 'requires a live iOS device/simulator toolchain (xcodebuild, devicectl) — manual hardware, not a CI runner capability',
|
||||
tracking: 'TODOS.md "skill-e2e-ios CI story" (device/runner decision)',
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,39 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
|
||||
import { runBin } from './run-bin';
|
||||
|
||||
describe('runBin', () => {
|
||||
test('captures status/stdout/stderr with utf-8 shaping', () => {
|
||||
const r = runBin('sh', ['-c', 'printf out; printf err >&2; exit 3']);
|
||||
expect(r).toEqual({ status: 3, stdout: 'out', stderr: 'err' });
|
||||
});
|
||||
|
||||
test('gstackHome sets both GSTACK_HOME and GSTACK_STATE_DIR (config precedence)', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'run-bin-'));
|
||||
try {
|
||||
const r = runBin('sh', ['-c', 'printf "%s|%s" "$GSTACK_HOME" "$GSTACK_STATE_DIR"'], { gstackHome: dir });
|
||||
expect(r.stdout).toBe(`${dir}|${dir}`);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('env undefined deletes a key; input feeds stdin; trim shapes output', () => {
|
||||
const r = runBin('sh', ['-c', 'cat; printf " padded "; test -z "$LANG" && printf noLANG >&2'], {
|
||||
env: { LANG: undefined },
|
||||
input: 'piped|',
|
||||
trim: true,
|
||||
});
|
||||
// trim shapes the ENDS of the whole stream; interior whitespace stays.
|
||||
expect(r.stdout).toBe('piped| padded');
|
||||
expect(r.stderr).toBe('noLANG');
|
||||
});
|
||||
|
||||
test('spawn failure yields -1, never a fake success', () => {
|
||||
const r = runBin('/definitely/not/a/binary');
|
||||
expect(r.status).toBe(-1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* Shared spawnSync wrapper for free unit tests that shell out to bin/
|
||||
* scripts. Before this helper, ~36 test files each carried a near-identical
|
||||
* local `run()` (spawnSync + utf-8 + {status, stdout, stderr} normalization)
|
||||
* differing only in env composition, cwd, and timeout — drift-prone copies
|
||||
* of one idea.
|
||||
*
|
||||
* Free-test-only by design: nothing under the paid globs should import this,
|
||||
* so it never becomes a de facto global touchfile (paid selection is owned
|
||||
* by test/helpers/e2e-helpers.ts and friends).
|
||||
*/
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
export interface RunBinResult {
|
||||
status: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}
|
||||
|
||||
export interface RunBinOptions {
|
||||
cwd?: string;
|
||||
/** Merged over process.env (an `undefined` value deletes the key). */
|
||||
env?: Record<string, string | undefined>;
|
||||
/**
|
||||
* Isolation shorthand: sets GSTACK_HOME + GSTACK_STATE_DIR (gstack-config
|
||||
* precedence is GSTACK_HOME > GSTACK_STATE_DIR > $HOME/.gstack, so both
|
||||
* must move to isolate from the operator's real ~/.gstack).
|
||||
*/
|
||||
gstackHome?: string;
|
||||
/** Also move $HOME (bins that write $HOME-anchored files, e.g. artifacts-remote pointers). */
|
||||
home?: string;
|
||||
input?: string;
|
||||
/** Default 60s — a wedged bin fails the test, never the shard wall. */
|
||||
timeoutMs?: number;
|
||||
maxBuffer?: number;
|
||||
/** Trim stdout/stderr (config-getter style bins). */
|
||||
trim?: boolean;
|
||||
}
|
||||
|
||||
export function runBin(command: string, args: string[] = [], opts: RunBinOptions = {}): RunBinResult {
|
||||
const env: Record<string, string | undefined> = { ...process.env, ...opts.env };
|
||||
if (opts.gstackHome !== undefined) {
|
||||
env.GSTACK_HOME = opts.gstackHome;
|
||||
env.GSTACK_STATE_DIR = opts.gstackHome;
|
||||
}
|
||||
if (opts.home !== undefined) env.HOME = opts.home;
|
||||
for (const key of Object.keys(env)) {
|
||||
if (env[key] === undefined) delete env[key];
|
||||
}
|
||||
|
||||
const result = spawnSync(command, args, {
|
||||
cwd: opts.cwd,
|
||||
env: env as Record<string, string>,
|
||||
encoding: 'utf-8',
|
||||
input: opts.input,
|
||||
timeout: opts.timeoutMs ?? 60_000,
|
||||
maxBuffer: opts.maxBuffer,
|
||||
});
|
||||
|
||||
const shape = (text: string | null | undefined): string => {
|
||||
const value = text ?? '';
|
||||
return opts.trim ? value.trim() : value;
|
||||
};
|
||||
return {
|
||||
// -1 for spawn failure/kill mirrors the strictest of the old locals: a
|
||||
// null status must never alias a real exit code.
|
||||
status: result.status ?? -1,
|
||||
stdout: shape(result.stdout),
|
||||
stderr: shape(result.stderr),
|
||||
};
|
||||
}
|
||||
+135
-130
@@ -22,8 +22,8 @@
|
||||
*/
|
||||
export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// Browse core (+ test-server dependency)
|
||||
'browse-basic': ['browse/src/**', 'browse/test/test-server.ts'],
|
||||
'browse-snapshot': ['browse/src/**', 'browse/test/test-server.ts'],
|
||||
'browse-basic': ['browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-bws.test.ts'],
|
||||
'browse-snapshot': ['browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-bws.test.ts'],
|
||||
|
||||
// Hermetic isolation canaries (hermetic-env.ts is also a GLOBAL touchfile;
|
||||
// these entries exist so the canaries themselves stay tier-classified)
|
||||
@@ -37,21 +37,21 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'first-task-scaffold': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-first-task-detect', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'test/skill-e2e-first-task-scaffold.test.ts', 'test/helpers/session-runner.ts'],
|
||||
|
||||
// SKILL.md setup + preamble (depend on ROOT SKILL.md + gen-skill-docs)
|
||||
'skillmd-setup-discovery': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'skillmd-no-local-binary': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'skillmd-outside-git': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'skillmd-setup-discovery': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'],
|
||||
'skillmd-no-local-binary': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'],
|
||||
'skillmd-outside-git': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'],
|
||||
|
||||
'session-awareness': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'operational-learning': ['scripts/resolvers/preamble.ts', 'bin/gstack-learnings-log'],
|
||||
'session-awareness': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'],
|
||||
'operational-learning': ['scripts/resolvers/preamble.ts', 'bin/gstack-learnings-log', 'test/skill-e2e-bws.test.ts'],
|
||||
|
||||
// QA (+ test-server dependency)
|
||||
'qa-quick': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts'],
|
||||
'qa-b6-static': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval.html', 'test/fixtures/qa-eval-ground-truth.json'],
|
||||
'qa-b7-spa': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-spa.html', 'test/fixtures/qa-eval-spa-ground-truth.json'],
|
||||
'qa-b8-checkout': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-checkout.html', 'test/fixtures/qa-eval-checkout-ground-truth.json'],
|
||||
'qa-only-no-fix': ['qa-only/**', 'qa/templates/**'],
|
||||
'qa-fix-loop': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts'],
|
||||
'qa-bootstrap': ['qa/**', 'ship/**'],
|
||||
'qa-quick': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts'],
|
||||
'qa-b6-static': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval.html', 'test/fixtures/qa-eval-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts'],
|
||||
'qa-b7-spa': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-spa.html', 'test/fixtures/qa-eval-spa-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts'],
|
||||
'qa-b8-checkout': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-checkout.html', 'test/fixtures/qa-eval-checkout-ground-truth.json', 'test/skill-e2e-qa-bugs.test.ts'],
|
||||
'qa-only-no-fix': ['qa-only/**', 'qa/templates/**', 'test/skill-e2e-qa-workflow.test.ts'],
|
||||
'qa-fix-loop': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/skill-e2e-qa-workflow.test.ts'],
|
||||
'qa-bootstrap': ['qa/**', 'ship/**', 'test/skill-e2e-qa-workflow.test.ts'],
|
||||
|
||||
// Review
|
||||
'review-sql-injection': ['review/**', 'test/fixtures/review-eval-vuln.rb', 'test/skill-e2e-review.test.ts'],
|
||||
@@ -60,29 +60,29 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'review-design-lite': ['review/**', 'test/fixtures/review-eval-design-slop.*', 'test/skill-e2e-review.test.ts'],
|
||||
|
||||
// Review Army (specialist dispatch)
|
||||
'review-army-migration-safety': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope'],
|
||||
'review-army-perf-n-plus-one': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope'],
|
||||
'review-army-delivery-audit': ['review/**', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-quality-score': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-json-findings': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-red-team': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-migration-safety': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope', 'test/skill-e2e-review-army.test.ts'],
|
||||
'review-army-perf-n-plus-one': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope', 'test/skill-e2e-review-army.test.ts'],
|
||||
'review-army-delivery-audit': ['review/**', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts'],
|
||||
'review-army-quality-score': ['review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts'],
|
||||
'review-army-json-findings': ['review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts'],
|
||||
'review-army-red-team': ['review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts'],
|
||||
'review-army-simplification': ['review/**', 'scripts/resolvers/review-army.ts', 'test/fixtures/review-army-overbuild.js', 'test/fixtures/review-army-lean-complete.js', 'test/skill-e2e-review-army.test.ts'],
|
||||
'review-army-simplification-precision': ['review/**', 'scripts/resolvers/review-army.ts', 'test/fixtures/review-army-overbuild.js', 'test/fixtures/review-army-lean-complete.js', 'test/skill-e2e-review-army.test.ts'],
|
||||
'review-army-consensus': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-consensus': ['review/**', 'scripts/resolvers/review-army.ts', 'test/skill-e2e-review-army.test.ts'],
|
||||
|
||||
// Office Hours
|
||||
'office-hours-spec-review': ['office-hours/**', 'scripts/gen-skill-docs.ts'],
|
||||
'office-hours-forcing-energy': ['office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts'],
|
||||
'office-hours-builder-wildness': ['office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts'],
|
||||
'office-hours-spec-review': ['office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'],
|
||||
'office-hours-forcing-energy': ['office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours.test.ts'],
|
||||
'office-hours-builder-wildness': ['office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours.test.ts'],
|
||||
|
||||
// Plan reviews
|
||||
'plan-ceo-review': ['plan-ceo-review/**'],
|
||||
'plan-ceo-review-selective': ['plan-ceo-review/**'],
|
||||
'plan-ceo-review-benefits': ['plan-ceo-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
'plan-ceo-review-expansion-energy': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts'],
|
||||
'plan-eng-review': ['plan-eng-review/**'],
|
||||
'plan-eng-review-artifact': ['plan-eng-review/**'],
|
||||
'plan-review-report': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
'plan-ceo-review': ['plan-ceo-review/**', 'test/skill-e2e-plan.test.ts'],
|
||||
'plan-ceo-review-selective': ['plan-ceo-review/**', 'test/skill-e2e-plan.test.ts'],
|
||||
'plan-ceo-review-benefits': ['plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'],
|
||||
'plan-ceo-review-expansion-energy': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan.test.ts'],
|
||||
'plan-eng-review': ['plan-eng-review/**', 'test/skill-e2e-plan.test.ts'],
|
||||
'plan-eng-review-artifact': ['plan-eng-review/**', 'test/skill-e2e-plan.test.ts'],
|
||||
'plan-review-report': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'],
|
||||
|
||||
// Plan-mode smoke tests — gate-tier safety regression tests. Each test file
|
||||
// contains TWO test cases as of v1.21: the baseline plan-mode case and the
|
||||
@@ -93,7 +93,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// regression test outcome between 'asked' and 'auto_decided'.
|
||||
'plan-ceo-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'],
|
||||
'plan-eng-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'],
|
||||
'plan-design-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts'],
|
||||
'plan-design-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts', 'test/skill-e2e-design.test.ts'],
|
||||
'plan-devex-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts'],
|
||||
// Covers ceo (preamble misfire) + eng/design (scope-gate bypass must not
|
||||
// fire outside plan mode) + the named-target exception case. 4 PTY runs;
|
||||
@@ -119,7 +119,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// written a never-ask preference, AUQ should still auto-decide rather than
|
||||
// surfacing the question. Touches the question-tuning + preference
|
||||
// infrastructure plus the resolvers that own the AUTO_DECIDE preamble.
|
||||
'auto-decide-preserved': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
'auto-decide-preserved': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-auto-decide-preserved.test.ts'],
|
||||
|
||||
// Conductor → prose decision brief (Conductor signal makes prose the default;
|
||||
// the PreToolUse hook denies the flaky tool). Touches the resolver that owns
|
||||
@@ -130,7 +130,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// Each one tests behavior the SDK harness can't observe (rendered TTY,
|
||||
// numbered-option lists, multi-phase ordering, idempotency state echo).
|
||||
'preamble-script-ab': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'plan-ceo-review/**', 'test/helpers/auq-sdk-capture.ts', 'test/skill-e2e-preamble-script-ab.test.ts'],
|
||||
'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts'],
|
||||
'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-ask-user-question-format-compliance.test.ts'],
|
||||
'auq-repetition-cut-ab': ['scripts/resolvers/preamble/generate-ask-user-format.ts', 'plan-ceo-review/**', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/llm-judge.ts', 'test/fixtures/auq-pre-cut-plan-ceo-review-SKILL.md', 'test/skill-e2e-auq-repetition-cut-ab.test.ts'],
|
||||
'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'],
|
||||
'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'],
|
||||
@@ -142,12 +142,12 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'tpa-absent-darwin': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
|
||||
'tpa-apple-ban': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
|
||||
'ship-section-loading': ['ship/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-ship-section-loading.test.ts'],
|
||||
'plan-ceo-section-loading': ['plan-ceo-review/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
|
||||
'plan-ceo-section-loading': ['plan-ceo-review/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-plan-ceo-review-section-loading.test.ts'],
|
||||
// Data-driven behavioral guard for the 'plan'/'prompt' carves (eng, design,
|
||||
// devex, office-hours + future PR2 carves). One file iterating CARVE_GUARDS;
|
||||
// the selector sets GSTACK_CARVE_SKILL=<name> to scope cost to the changed
|
||||
// skill (D-CODEX A). Touching the registry/helper or sections.ts runs all.
|
||||
'carve-section-loading': ['design-html/**', 'design-shotgun/**', 'qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
|
||||
'carve-section-loading': ['design-html/**', 'design-shotgun/**', 'qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/carve-section-loading.test.ts'],
|
||||
'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-autoplan-chain.test.ts'],
|
||||
'e2e-harness-audit': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
|
||||
@@ -193,18 +193,18 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
// AskUserQuestion format regression (RECOMMENDATION + Completeness: N/10)
|
||||
// Fires when either template OR the two preamble resolvers change.
|
||||
'plan-ceo-review-format-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'],
|
||||
'plan-ceo-review-format-approach': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'],
|
||||
'plan-eng-review-format-coverage': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'],
|
||||
'plan-eng-review-format-kind': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'],
|
||||
'plan-ceo-review-format-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts'],
|
||||
'plan-ceo-review-format-approach': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts'],
|
||||
'plan-eng-review-format-coverage': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts'],
|
||||
'plan-eng-review-format-kind': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan-format.test.ts'],
|
||||
|
||||
// v1.7.0.0 Pros/Cons format cadence + format + negative-escape evals.
|
||||
// Dependencies: same as format-mode + the 4 plan-review templates + overlay.
|
||||
// All periodic-tier (non-deterministic Opus 4.7 behavior).
|
||||
'plan-ceo-review-prosons-cadence': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'plan-review-prosons-format': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'plan-review-prosons-hardstop-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'plan-review-prosons-neutral-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'plan-ceo-review-prosons-cadence': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/skill-e2e-plan-prosons.test.ts'],
|
||||
'plan-review-prosons-format': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/skill-e2e-plan-prosons.test.ts'],
|
||||
'plan-review-prosons-hardstop-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/skill-e2e-plan-prosons.test.ts'],
|
||||
'plan-review-prosons-neutral-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/skill-e2e-plan-prosons.test.ts'],
|
||||
|
||||
// Expanded coverage (CT3) — 6 non-plan-review skills inherit Pros/Cons via preamble
|
||||
'ship-prosons-format': ['ship/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
@@ -216,24 +216,24 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'document-release-prosons-format': ['document-release/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
|
||||
// /plan-tune (v1 observational)
|
||||
'plan-tune-inspect': ['plan-tune/**', 'scripts/question-registry.ts', 'scripts/psychographic-signals.ts', 'scripts/one-way-doors.ts', 'bin/gstack-question-log', 'bin/gstack-question-preference', 'bin/gstack-developer-profile'],
|
||||
'plan-tune-inspect': ['plan-tune/**', 'scripts/question-registry.ts', 'scripts/psychographic-signals.ts', 'scripts/one-way-doors.ts', 'bin/gstack-question-log', 'bin/gstack-question-preference', 'bin/gstack-developer-profile', 'test/skill-e2e-plan-tune.test.ts'],
|
||||
|
||||
// /plan-tune cathedral (T16 — 5 E2E scenarios, all gate per D12)
|
||||
'plan-tune-hook-capture': ['hosts/claude/hooks/**', 'bin/gstack-question-log', 'bin/gstack-developer-profile', 'plan-tune/**'],
|
||||
'plan-tune-enforcement': ['hosts/claude/hooks/**', 'bin/gstack-question-preference', 'scripts/question-registry.ts'],
|
||||
'plan-tune-annotation': ['hosts/claude/hooks/**', 'scripts/declared-annotation.ts', 'scripts/psychographic-signals.ts', 'scripts/question-registry.ts'],
|
||||
'plan-tune-codex-import': ['bin/gstack-codex-session-import', 'bin/gstack-question-log', 'docs/spikes/codex-session-format.md'],
|
||||
'plan-tune-dream-cycle': ['bin/gstack-distill-free-text', 'bin/gstack-distill-apply', 'hosts/claude/hooks/**', 'plan-tune/**'],
|
||||
'plan-tune-hook-capture': ['hosts/claude/hooks/**', 'bin/gstack-question-log', 'bin/gstack-developer-profile', 'plan-tune/**', 'test/skill-e2e-plan-tune-cathedral.test.ts'],
|
||||
'plan-tune-enforcement': ['hosts/claude/hooks/**', 'bin/gstack-question-preference', 'scripts/question-registry.ts', 'test/skill-e2e-plan-tune-cathedral.test.ts'],
|
||||
'plan-tune-annotation': ['hosts/claude/hooks/**', 'scripts/declared-annotation.ts', 'scripts/psychographic-signals.ts', 'scripts/question-registry.ts', 'test/skill-e2e-plan-tune-cathedral.test.ts'],
|
||||
'plan-tune-codex-import': ['bin/gstack-codex-session-import', 'bin/gstack-question-log', 'docs/spikes/codex-session-format.md', 'test/skill-e2e-plan-tune-cathedral.test.ts'],
|
||||
'plan-tune-dream-cycle': ['bin/gstack-distill-free-text', 'bin/gstack-distill-apply', 'hosts/claude/hooks/**', 'plan-tune/**', 'test/skill-e2e-plan-tune-cathedral.test.ts'],
|
||||
|
||||
// Codex offering verification
|
||||
'codex-offered-office-hours': ['office-hours/**', 'scripts/gen-skill-docs.ts'],
|
||||
'codex-offered-ceo-review': ['plan-ceo-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
'codex-offered-design-review': ['plan-design-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
'codex-offered-eng-review': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
'codex-offered-office-hours': ['office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'],
|
||||
'codex-offered-ceo-review': ['plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'],
|
||||
'codex-offered-design-review': ['plan-design-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'],
|
||||
'codex-offered-eng-review': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts'],
|
||||
|
||||
// Ship
|
||||
'ship-base-branch': ['ship/**', 'bin/gstack-repo-mode', 'test/skill-e2e-review-attribution.test.ts'],
|
||||
'ship-local-workflow': ['ship/**', 'scripts/gen-skill-docs.ts'],
|
||||
'ship-local-workflow': ['ship/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-workflow.test.ts'],
|
||||
'review-dashboard-via': ['ship/**', 'scripts/resolvers/review.ts', 'codex/**', 'autoplan/**', 'land-and-deploy/**', 'test/skill-e2e-review-attribution.test.ts'],
|
||||
|
||||
// Retro
|
||||
@@ -244,51 +244,51 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'global-discover': ['bin/gstack-global-discover.ts', 'test/global-discover.test.ts'],
|
||||
|
||||
// CSO
|
||||
'cso-full-audit': ['cso/**'],
|
||||
'cso-diff-mode': ['cso/**'],
|
||||
'cso-infra-scope': ['cso/**'],
|
||||
'cso-full-audit': ['cso/**', 'test/skill-e2e-cso.test.ts'],
|
||||
'cso-diff-mode': ['cso/**', 'test/skill-e2e-cso.test.ts'],
|
||||
'cso-infra-scope': ['cso/**', 'test/skill-e2e-cso.test.ts'],
|
||||
|
||||
// Learnings
|
||||
'learnings-show': ['learn/**', 'bin/gstack-learnings-search', 'bin/gstack-learnings-log', 'scripts/resolvers/learnings.ts'],
|
||||
'learnings-show': ['learn/**', 'bin/gstack-learnings-search', 'bin/gstack-learnings-log', 'scripts/resolvers/learnings.ts', 'test/skill-e2e-learnings.test.ts'],
|
||||
|
||||
// Session Intelligence (timeline, context recovery, /context-save + /context-restore)
|
||||
'timeline-event-flow': ['bin/gstack-timeline-log', 'bin/gstack-timeline-read'],
|
||||
'context-recovery-artifacts': ['scripts/resolvers/preamble.ts', 'bin/gstack-timeline-log', 'bin/gstack-slug', 'learn/**'],
|
||||
'context-save-writes-file': ['context-save/**', 'bin/gstack-slug'],
|
||||
'context-restore-loads-latest': ['context-restore/**', 'bin/gstack-slug'],
|
||||
'timeline-event-flow': ['bin/gstack-timeline-log', 'bin/gstack-timeline-read', 'test/skill-e2e-session-intelligence.test.ts'],
|
||||
'context-recovery-artifacts': ['scripts/resolvers/preamble.ts', 'bin/gstack-timeline-log', 'bin/gstack-slug', 'learn/**', 'test/skill-e2e-session-intelligence.test.ts'],
|
||||
'context-save-writes-file': ['context-save/**', 'bin/gstack-slug', 'test/skill-e2e-session-intelligence.test.ts'],
|
||||
'context-restore-loads-latest': ['context-restore/**', 'bin/gstack-slug', 'test/skill-e2e-session-intelligence.test.ts'],
|
||||
|
||||
// Context skills E2E (live-fire, Skill-tool routing path) — see
|
||||
// test/skill-e2e-context-skills.test.ts. These are periodic-tier because
|
||||
// each one spawns claude -p and costs ~$0.20-$0.40. Collectively they
|
||||
// verify the thing the /checkpoint → /context-save rename was for.
|
||||
'context-save-routing': ['context-save/**', 'scripts/resolvers/preamble.ts'],
|
||||
'context-save-then-restore-roundtrip': ['context-save/**', 'context-restore/**', 'bin/gstack-slug'],
|
||||
'context-restore-fragment-match': ['context-restore/**'],
|
||||
'context-restore-empty-state': ['context-restore/**'],
|
||||
'context-restore-list-delegates': ['context-restore/**'],
|
||||
'context-restore-legacy-compat': ['context-restore/**'],
|
||||
'context-save-list-current-branch': ['context-save/**'],
|
||||
'context-save-list-all-branches': ['context-save/**'],
|
||||
'context-save-routing': ['context-save/**', 'scripts/resolvers/preamble.ts', 'test/skill-e2e-context-skills.test.ts'],
|
||||
'context-save-then-restore-roundtrip': ['context-save/**', 'context-restore/**', 'bin/gstack-slug', 'test/skill-e2e-context-skills.test.ts'],
|
||||
'context-restore-fragment-match': ['context-restore/**', 'test/skill-e2e-context-skills.test.ts'],
|
||||
'context-restore-empty-state': ['context-restore/**', 'test/skill-e2e-context-skills.test.ts'],
|
||||
'context-restore-list-delegates': ['context-restore/**', 'test/skill-e2e-context-skills.test.ts'],
|
||||
'context-restore-legacy-compat': ['context-restore/**', 'test/skill-e2e-context-skills.test.ts'],
|
||||
'context-save-list-current-branch': ['context-save/**', 'test/skill-e2e-context-skills.test.ts'],
|
||||
'context-save-list-all-branches': ['context-save/**', 'test/skill-e2e-context-skills.test.ts'],
|
||||
|
||||
// Document-release
|
||||
'document-release': ['document-release/**'],
|
||||
'document-release': ['document-release/**', 'test/skill-e2e-workflow.test.ts'],
|
||||
|
||||
// Codex (Claude E2E — tests /codex skill via Claude)
|
||||
'codex-review': ['codex/**'],
|
||||
'codex-review': ['codex/**', 'test/skill-e2e-workflow.test.ts'],
|
||||
|
||||
// Codex E2E (tests skills via Codex CLI + worktree)
|
||||
'codex-discover-skill': ['codex/**', '.agents/skills/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts'],
|
||||
'codex-review-findings': ['review/**', '.agents/skills/gstack-review/**', 'codex/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts'],
|
||||
'codex-discover-skill': ['codex/**', '.agents/skills/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts', 'test/codex-e2e.test.ts'],
|
||||
'codex-review-findings': ['review/**', '.agents/skills/gstack-review/**', 'codex/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts', 'test/codex-e2e.test.ts'],
|
||||
|
||||
// GPT-5.6 Sol scope-termination E2E (Codex CLI, full generated investigate skill)
|
||||
'codex-sol-scope-termination': ['model-overlays/gpt-5.6-sol.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts', 'scripts/resolvers/preamble/**', 'investigate/**', 'test/helpers/codex-session-runner.ts', 'test/codex-e2e-sol-scope.test.ts'],
|
||||
|
||||
// Gemini E2E — smoke test only (Gemini gets lost in worktrees on complex tasks)
|
||||
'gemini-smoke': ['.agents/skills/**', 'test/helpers/gemini-session-runner.ts', 'lib/worktree.ts'],
|
||||
'gemini-smoke': ['.agents/skills/**', 'test/helpers/gemini-session-runner.ts', 'lib/worktree.ts', 'test/gemini-e2e.test.ts'],
|
||||
|
||||
|
||||
// Coverage audit (shared fixture) + triage + gates
|
||||
'ship-coverage-audit': ['ship/**', 'test/fixtures/coverage-audit-fixture.ts', 'bin/gstack-repo-mode'],
|
||||
'ship-coverage-audit': ['ship/**', 'test/fixtures/coverage-audit-fixture.ts', 'bin/gstack-repo-mode', 'test/skill-e2e-workflow.test.ts'],
|
||||
'review-coverage-audit': ['review/**', 'test/fixtures/coverage-audit-fixture.ts', 'test/skill-e2e-coverage-audit.test.ts'],
|
||||
'plan-eng-coverage-audit': ['plan-eng-review/**', 'test/fixtures/coverage-audit-fixture.ts', 'test/skill-e2e-coverage-audit.test.ts'],
|
||||
'ship-triage': ['ship/**', 'bin/gstack-repo-mode', 'test/skill-e2e-triage.test.ts'],
|
||||
@@ -300,12 +300,12 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'review-plan-completion': ['review/**', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Design
|
||||
'design-consultation-core': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/helpers/llm-judge.ts'],
|
||||
'design-consultation-existing': ['design-consultation/**', 'scripts/gen-skill-docs.ts'],
|
||||
'design-consultation-research': ['design-consultation/**', 'scripts/gen-skill-docs.ts'],
|
||||
'design-consultation-preview': ['design-consultation/**', 'scripts/gen-skill-docs.ts'],
|
||||
'plan-design-review-no-ui-scope': ['plan-design-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
'design-review-fix': ['design-review/**', 'browse/src/**', 'scripts/gen-skill-docs.ts'],
|
||||
'design-consultation-core': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-design.test.ts'],
|
||||
'design-consultation-existing': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
|
||||
'design-consultation-research': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
|
||||
'design-consultation-preview': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
|
||||
'plan-design-review-no-ui-scope': ['plan-design-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
|
||||
'design-review-fix': ['design-review/**', 'browse/src/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-design.test.ts'],
|
||||
|
||||
// Design Shotgun
|
||||
'design-shotgun-path': ['design-shotgun/**', 'design/src/**', 'scripts/resolvers/design.ts'],
|
||||
@@ -314,24 +314,24 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
// /diagram (diagram-render bundle consumers). Triplet = deterministic
|
||||
// functional (gate); authoring quality = LLM-judged benchmark (periodic).
|
||||
'diagram-triplet': ['diagram/**', 'lib/diagram-render/**', 'browse/src/write-commands.ts', 'browse/src/read-commands.ts'],
|
||||
'diagram-authoring-quality': ['diagram/**', 'lib/diagram-render/**', 'test/helpers/llm-judge.ts'],
|
||||
'diagram-triplet': ['diagram/**', 'lib/diagram-render/**', 'browse/src/write-commands.ts', 'browse/src/read-commands.ts', 'test/skill-e2e-diagram.test.ts'],
|
||||
'diagram-authoring-quality': ['diagram/**', 'lib/diagram-render/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-diagram.test.ts'],
|
||||
|
||||
// gstack-upgrade
|
||||
'gstack-upgrade-happy-path': ['gstack-upgrade/**'],
|
||||
'gstack-upgrade-happy-path': ['gstack-upgrade/**', 'test/skill-e2e-workflow.test.ts'],
|
||||
|
||||
// Deploy skills
|
||||
'land-and-deploy-workflow': ['land-and-deploy/**', 'scripts/gen-skill-docs.ts'],
|
||||
'land-and-deploy-first-run': ['land-and-deploy/**', 'scripts/gen-skill-docs.ts', 'bin/gstack-slug'],
|
||||
'land-and-deploy-review-gate': ['land-and-deploy/**', 'bin/gstack-review-read'],
|
||||
'canary-workflow': ['canary/**', 'browse/src/**'],
|
||||
'benchmark-workflow': ['benchmark/**', 'browse/src/**'],
|
||||
'setup-deploy-workflow': ['setup-deploy/**', 'scripts/gen-skill-docs.ts'],
|
||||
'land-and-deploy-workflow': ['land-and-deploy/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-deploy.test.ts'],
|
||||
'land-and-deploy-first-run': ['land-and-deploy/**', 'scripts/gen-skill-docs.ts', 'bin/gstack-slug', 'test/skill-e2e-deploy.test.ts'],
|
||||
'land-and-deploy-review-gate': ['land-and-deploy/**', 'bin/gstack-review-read', 'test/skill-e2e-deploy.test.ts'],
|
||||
'canary-workflow': ['canary/**', 'browse/src/**', 'test/skill-e2e-deploy.test.ts'],
|
||||
'benchmark-workflow': ['benchmark/**', 'browse/src/**', 'test/skill-e2e-deploy.test.ts'],
|
||||
'setup-deploy-workflow': ['setup-deploy/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-deploy.test.ts'],
|
||||
|
||||
|
||||
// Autoplan
|
||||
'autoplan-core': ['autoplan/**', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**'],
|
||||
'autoplan-dual-voice': ['autoplan/**', 'codex/**', 'bin/gstack-codex-probe', 'scripts/resolvers/review.ts', 'scripts/resolvers/design.ts'],
|
||||
'autoplan-dual-voice': ['autoplan/**', 'codex/**', 'bin/gstack-codex-probe', 'scripts/resolvers/review.ts', 'scripts/resolvers/design.ts', 'test/skill-e2e-autoplan-dual-voice.test.ts'],
|
||||
|
||||
// Multi-provider benchmark adapters — live API smoke against real claude/codex/gemini CLIs
|
||||
'benchmark-providers-live': ['bin/gstack-model-benchmark', 'test/helpers/providers/**', 'test/helpers/benchmark-runner.ts', 'test/helpers/pricing.ts', 'test/skill-e2e-benchmark-providers.test.ts'],
|
||||
@@ -344,41 +344,46 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'scrape-match-path': [
|
||||
'scrape/**', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts',
|
||||
'browser-skills/hackernews-frontpage/**',
|
||||
'test/skill-e2e-skillify.test.ts',
|
||||
],
|
||||
'scrape-prototype-path': [
|
||||
'scrape/**', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts',
|
||||
'test/skill-e2e-skillify.test.ts',
|
||||
],
|
||||
'skillify-happy-path': [
|
||||
'skillify/**', 'scrape/**', 'browse/src/browser-skill-write.ts',
|
||||
'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts',
|
||||
'test/skill-e2e-skillify.test.ts',
|
||||
],
|
||||
'skillify-provenance-refusal': [
|
||||
'skillify/**', 'browse/src/browser-skill-write.ts',
|
||||
'test/skill-e2e-skillify.test.ts',
|
||||
],
|
||||
'skillify-approval-reject': [
|
||||
'skillify/**', 'scrape/**', 'browse/src/browser-skill-write.ts',
|
||||
'test/skill-e2e-skillify.test.ts',
|
||||
],
|
||||
|
||||
// Skill routing — journey-stage tests (depend on ALL skill descriptions)
|
||||
'journey-ideation': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-plan-eng': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-debug': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-code-review': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-ship': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-docs': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-retro': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-design-system': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-visual-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-ideation': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
|
||||
'journey-plan-eng': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
|
||||
'journey-debug': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
|
||||
'journey-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
|
||||
'journey-code-review': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
|
||||
'journey-ship': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
|
||||
'journey-docs': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
|
||||
'journey-retro': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
|
||||
'journey-design-system': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
|
||||
'journey-visual-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts'],
|
||||
|
||||
// Opus 4.7 behavior evals — keys match testName: values in the test file.
|
||||
// Routing sub-tests use template literal `routing-${c.name}` testNames,
|
||||
// which the touchfile completeness scanner skips; they inherit selection
|
||||
// from the file-level touchfile entry via GLOBAL_TOUCHFILES.
|
||||
'fanout-arm-overlay-on':
|
||||
['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts'],
|
||||
['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-opus-47.test.ts'],
|
||||
'fanout-arm-overlay-off':
|
||||
['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts'],
|
||||
['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-opus-47.test.ts'],
|
||||
|
||||
// Overlay efficacy harness (SDK) — measures whether overlay nudges change
|
||||
// behavior under @anthropic-ai/claude-agent-sdk (closer to real Claude Code
|
||||
@@ -809,49 +814,49 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
* LLM-judge test touchfiles — keyed by test description string.
|
||||
*/
|
||||
export const LLM_JUDGE_TOUCHFILES: Record<string, string[]> = {
|
||||
'command reference table': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts'],
|
||||
'snapshot flags reference': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts'],
|
||||
'browse/SKILL.md reference': ['browse/sections/**', 'browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**'],
|
||||
'setup block': ['SKILL.md', 'SKILL.md.tmpl'],
|
||||
'regression vs baseline': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json'],
|
||||
'qa/SKILL.md workflow': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl'],
|
||||
'qa/SKILL.md health rubric': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl'],
|
||||
'qa/SKILL.md anti-refusal': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'],
|
||||
'cross-skill greptile consistency': ['review/SKILL.md', 'review/SKILL.md.tmpl', 'ship/SKILL.md', 'ship/SKILL.md.tmpl', 'review/greptile-triage.md', 'retro/SKILL.md', 'retro/SKILL.md.tmpl'],
|
||||
'baseline score pinning': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json'],
|
||||
'command reference table': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/skill-llm-eval.test.ts'],
|
||||
'snapshot flags reference': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts', 'test/skill-llm-eval.test.ts'],
|
||||
'browse/SKILL.md reference': ['browse/sections/**', 'browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**', 'test/skill-llm-eval.test.ts'],
|
||||
'setup block': ['SKILL.md', 'SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'regression vs baseline': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json', 'test/skill-llm-eval.test.ts'],
|
||||
'qa/SKILL.md workflow': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'qa/SKILL.md health rubric': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'qa/SKILL.md anti-refusal': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'cross-skill greptile consistency': ['review/SKILL.md', 'review/SKILL.md.tmpl', 'ship/SKILL.md', 'ship/SKILL.md.tmpl', 'review/greptile-triage.md', 'retro/SKILL.md', 'retro/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'baseline score pinning': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json', 'test/skill-llm-eval.test.ts'],
|
||||
|
||||
// Ship & Release
|
||||
'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl'],
|
||||
'document-release/SKILL.md workflow': ['document-release/SKILL.md', 'document-release/SKILL.md.tmpl'],
|
||||
'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'document-release/SKILL.md workflow': ['document-release/SKILL.md', 'document-release/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
|
||||
// Plan Reviews
|
||||
'plan-ceo-review/SKILL.md modes': ['plan-ceo-review/SKILL.md', 'plan-ceo-review/SKILL.md.tmpl'],
|
||||
'plan-eng-review/SKILL.md sections': ['plan-eng-review/SKILL.md', 'plan-eng-review/SKILL.md.tmpl'],
|
||||
'plan-ceo-review/SKILL.md modes': ['plan-ceo-review/SKILL.md', 'plan-ceo-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'plan-eng-review/SKILL.md sections': ['plan-eng-review/SKILL.md', 'plan-eng-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
|
||||
// /spec authored-spec quality (paid LLM-judge — periodic-tier).
|
||||
'plan-design-review/SKILL.md passes': ['plan-design-review/SKILL.md', 'plan-design-review/SKILL.md.tmpl'],
|
||||
'plan-design-review/SKILL.md passes': ['plan-design-review/SKILL.md', 'plan-design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
|
||||
// Design skills
|
||||
'design-review/SKILL.md fix loop': ['design-review/SKILL.md', 'design-review/SKILL.md.tmpl'],
|
||||
'design-consultation/SKILL.md research': ['design-consultation/SKILL.md', 'design-consultation/SKILL.md.tmpl'],
|
||||
'design-review/SKILL.md fix loop': ['design-review/SKILL.md', 'design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'design-consultation/SKILL.md research': ['design-consultation/SKILL.md', 'design-consultation/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
|
||||
// Office Hours
|
||||
'office-hours/SKILL.md spec review': ['office-hours/SKILL.md', 'office-hours/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'office-hours/SKILL.md design sketch': ['office-hours/SKILL.md', 'office-hours/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Deploy skills
|
||||
'land-and-deploy/SKILL.md workflow': ['land-and-deploy/SKILL.md', 'land-and-deploy/SKILL.md.tmpl', 'land-and-deploy/sections/**'],
|
||||
'canary/SKILL.md monitoring loop': ['canary/SKILL.md', 'canary/SKILL.md.tmpl'],
|
||||
'benchmark/SKILL.md perf collection': ['benchmark/SKILL.md', 'benchmark/SKILL.md.tmpl'],
|
||||
'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl'],
|
||||
'land-and-deploy/SKILL.md workflow': ['land-and-deploy/SKILL.md', 'land-and-deploy/SKILL.md.tmpl', 'land-and-deploy/sections/**', 'test/skill-llm-eval.test.ts'],
|
||||
'canary/SKILL.md monitoring loop': ['canary/SKILL.md', 'canary/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'benchmark/SKILL.md perf collection': ['benchmark/SKILL.md', 'benchmark/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
|
||||
// Other skills
|
||||
'retro/SKILL.md instructions': ['retro/sections/**', 'retro/SKILL.md', 'retro/SKILL.md.tmpl'],
|
||||
'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'],
|
||||
'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl'],
|
||||
'retro/SKILL.md instructions': ['retro/sections/**', 'retro/SKILL.md', 'retro/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
|
||||
// Voice directive
|
||||
'voice directive tone': ['scripts/resolvers/preamble.ts', 'review/SKILL.md', 'review/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'voice directive tone': ['scripts/resolvers/preamble.ts', 'review/SKILL.md', 'review/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-llm-eval.test.ts'],
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
+31
-21
@@ -3,8 +3,9 @@
|
||||
* host-config-export.ts, and golden-file regression checks.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll } from 'bun:test';
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { validateHostConfig, validateAllConfigs, type HostConfig } from '../scripts/host-config';
|
||||
import {
|
||||
@@ -428,34 +429,43 @@ describe('host-config-export.ts CLI', () => {
|
||||
describe('golden-file regression', () => {
|
||||
const GOLDEN_DIR = path.join(ROOT, 'test', 'fixtures', 'golden');
|
||||
|
||||
// #2532: the codex/factory goldens read gitignored .agents/ and .factory/
|
||||
// artifacts that only gen-skill-docs.test.ts (a serial tree-mutating file)
|
||||
// produces. On a clean clone — or when this file runs in isolation — those
|
||||
// dirs don't exist and the goldens fail with ENOENT, an order dependency,
|
||||
// not a regression. Self-provision: generate a host's artifacts iff its
|
||||
// ship SKILL.md is missing. Existing artifacts are never overwritten here,
|
||||
// so a genuinely stale artifact still fails the golden (that is the test's
|
||||
// job; freshness enforcement lives in gen-skill-docs.test.ts).
|
||||
// #2532 successor: the codex/factory goldens used to read gitignored
|
||||
// .agents/ and .factory/ artifacts "produced by gen-skill-docs.test.ts" —
|
||||
// an inter-test ordering dependency that failed with ENOENT on a clean
|
||||
// clone or when this file ran in isolation. Severed: this describe
|
||||
// UNCONDITIONALLY renders both hosts into its own --out-dir in beforeAll
|
||||
// and reads its goldens only from that render — no when-missing check, no
|
||||
// live-tree reads for the gitignored artifacts, no dependence on what any
|
||||
// other test left on disk. Comparing a FRESH render to the golden is also
|
||||
// strictly deterministic: a stale on-disk artifact can no longer mask (or
|
||||
// fake) a generator regression.
|
||||
const GOLDEN_OUT = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-golden-out-'));
|
||||
|
||||
beforeAll(() => {
|
||||
const hostArtifacts: Array<[string, string]> = [
|
||||
['codex', path.join(ROOT, '.agents', 'skills', 'gstack-ship', 'SKILL.md')],
|
||||
['factory', path.join(ROOT, '.factory', 'skills', 'gstack-ship', 'SKILL.md')],
|
||||
];
|
||||
for (const [host, artifact] of hostArtifacts) {
|
||||
if (fs.existsSync(artifact)) continue;
|
||||
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', host], {
|
||||
cwd: ROOT,
|
||||
});
|
||||
for (const host of ['codex', 'factory']) {
|
||||
const result = Bun.spawnSync(
|
||||
['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', host, '--out-dir', GOLDEN_OUT],
|
||||
{ cwd: ROOT },
|
||||
);
|
||||
if (result.exitCode !== 0) {
|
||||
throw new Error(
|
||||
`golden-file beforeAll: gen-skill-docs --host ${host} failed (exit ${result.exitCode}):\n`
|
||||
`golden-file beforeAll: gen-skill-docs --host ${host} --out-dir failed (exit ${result.exitCode}):\n`
|
||||
+ result.stderr.toString(),
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
fs.rmSync(GOLDEN_OUT, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('Claude ship skill matches golden baseline', () => {
|
||||
// Deliberately reads the TRACKED ship/SKILL.md (a read, not a write):
|
||||
// the claude golden pins the committed render. Freshness of the tracked
|
||||
// tree vs the templates is enforced by gen-skill-docs.test.ts. (An
|
||||
// out-dir claude render would NOT byte-match this golden — --out-dir
|
||||
// repoints section-base paths into the render by design.)
|
||||
const golden = fs.readFileSync(path.join(GOLDEN_DIR, 'claude-ship-SKILL.md'), 'utf-8');
|
||||
const current = fs.readFileSync(path.join(ROOT, 'ship', 'SKILL.md'), 'utf-8');
|
||||
expect(current).toBe(golden);
|
||||
@@ -463,13 +473,13 @@ describe('golden-file regression', () => {
|
||||
|
||||
test('Codex ship skill matches golden baseline', () => {
|
||||
const golden = fs.readFileSync(path.join(GOLDEN_DIR, 'codex-ship-SKILL.md'), 'utf-8');
|
||||
const current = fs.readFileSync(path.join(ROOT, '.agents', 'skills', 'gstack-ship', 'SKILL.md'), 'utf-8');
|
||||
const current = fs.readFileSync(path.join(GOLDEN_OUT, '.agents', 'skills', 'gstack-ship', 'SKILL.md'), 'utf-8');
|
||||
expect(current).toBe(golden);
|
||||
});
|
||||
|
||||
test('Factory ship skill matches golden baseline', () => {
|
||||
const golden = fs.readFileSync(path.join(GOLDEN_DIR, 'factory-ship-SKILL.md'), 'utf-8');
|
||||
const current = fs.readFileSync(path.join(ROOT, '.factory', 'skills', 'gstack-ship', 'SKILL.md'), 'utf-8');
|
||||
const current = fs.readFileSync(path.join(GOLDEN_OUT, '.factory', 'skills', 'gstack-ship', 'SKILL.md'), 'utf-8');
|
||||
expect(current).toBe(golden);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
*/
|
||||
|
||||
import { expect } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { judgeRecommendation } from './helpers/llm-judge';
|
||||
import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers';
|
||||
|
||||
@@ -181,5 +182,5 @@ Net: ...`);
|
||||
`[hedge:${label}] expected commits=false; got ${score.commits}. text="${text}"`,
|
||||
).toBe(false);
|
||||
}
|
||||
}, 240_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
/**
|
||||
* No paid-gated test file may sit outside PAID_TEST_GLOBS.
|
||||
*
|
||||
* The orphan class this kills (found 2026-08): a file whose source gates on
|
||||
* EVALS/tier (so the free suite loads it as describe.skip) but whose NAME
|
||||
* doesn't match the paid globs (so no paid lane ever selects it) can never
|
||||
* execute anywhere — forever, silently. Four files were in that state
|
||||
* (codex-e2e-plan-format, codex-e2e-recommendation-substance,
|
||||
* llm-judge-recommendation, carve-section-loading), and the tripwire built
|
||||
* for the adjacent class (test/evals-workflow-matrix.test.ts) couldn't see
|
||||
* them because it filters on isPaidTestFile() FIRST.
|
||||
*
|
||||
* Detection is over source text, so meta-tests and helpers that mention the
|
||||
* gate patterns need reasoned exemptions (same convention as
|
||||
* test/egress-receipt-wiring.test.ts's SCANNER_EXEMPT).
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
import { isPaidTestFile } from './helpers/paid-test-set';
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
|
||||
/** Files that legitimately mention gate patterns without being paid tests. */
|
||||
const SCANNER_EXEMPT = new Map<string, string>([
|
||||
// The gate helpers themselves and their free unit tests:
|
||||
['test/helpers/e2e-gate.ts', 'defines the gate predicates'],
|
||||
// Meta-tests that quote gate-pattern strings to test classification:
|
||||
['test/helpers/e2e-gate.unit.test.ts', 'free unit test OF the gate predicates (env stubbed)'],
|
||||
['test/paid-shards.test.ts', 'quotes tier-guard strings as classification fixtures'],
|
||||
['test/evals-workflow-matrix.test.ts', 'parses tier guards out of matrix files'],
|
||||
['test/e2e-tier-alignment.test.ts', 'parses tier guards to enforce alignment'],
|
||||
['test/paid-orphan-tripwire.test.ts', 'this scanner'],
|
||||
]);
|
||||
|
||||
/**
|
||||
* Source shapes that mean "this file self-gates on the paid env":
|
||||
* the shared helpers, or a direct EVALS/EVALS_TIER env read.
|
||||
*/
|
||||
const GATE_PATTERNS = [
|
||||
/\bdescribeE2ETier\s*\(/,
|
||||
/\be2eTierEnabled\s*\(/,
|
||||
/process\.env\.EVALS\b/,
|
||||
];
|
||||
|
||||
function trackedTestFiles(): string[] {
|
||||
const out = spawnSync('git', ['ls-files', '*.test.ts'], { cwd: ROOT, encoding: 'utf-8' });
|
||||
if (out.status !== 0) throw new Error(`git ls-files failed: ${out.stderr}`);
|
||||
return out.stdout.split('\n').filter(Boolean);
|
||||
}
|
||||
|
||||
describe('paid orphan tripwire', () => {
|
||||
test('every EVALS/tier-gated test file is inside PAID_TEST_GLOBS (or exempt with a reason)', () => {
|
||||
const files = trackedTestFiles();
|
||||
expect(files.length).toBeGreaterThan(100); // scan-rot guard
|
||||
|
||||
const orphans: string[] = [];
|
||||
for (const rel of files) {
|
||||
if (isPaidTestFile(rel)) continue;
|
||||
if (SCANNER_EXEMPT.has(rel)) continue;
|
||||
const source = fs.readFileSync(path.join(ROOT, rel), 'utf-8');
|
||||
const hit = GATE_PATTERNS.find((p) => p.test(source));
|
||||
if (hit) orphans.push(`${rel} (matches ${hit})`);
|
||||
}
|
||||
expect(orphans,
|
||||
'paid-gated test files OUTSIDE the paid globs can never run in any lane. '
|
||||
+ 'Fix: extend PAID_TEST_GLOBS in test/helpers/paid-test-set.ts (and mirror '
|
||||
+ 'package.json), or add a reasoned SCANNER_EXEMPT entry if the file only '
|
||||
+ `mentions the patterns:\n${orphans.join('\n')}`,
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
test('exemption entries stay real (stale entries must be deleted)', () => {
|
||||
for (const [rel] of SCANNER_EXEMPT) {
|
||||
expect(fs.existsSync(path.join(ROOT, rel)), `stale SCANNER_EXEMPT entry: ${rel}`).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,190 @@
|
||||
/**
|
||||
* Planner/executor/report contract for the re-platformed paid CI lane.
|
||||
*
|
||||
* The classes these pin (each was a live CI failure mode of the old
|
||||
* hand-enumerated matrix, or a review-identified risk of the migration):
|
||||
* - per-slice selector divergence → ONE planner manifest, executors consume
|
||||
* - hollow lanes → a slice with no artifact is a FAILURE, not an absence
|
||||
* - hollow shards → EVALS_ALL + exit 0 + zero executed tests ≠ pass
|
||||
* - retry parity → the old matrix rows' earned `retries: 2` survive as a
|
||||
* literals map, not folklore
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
import {
|
||||
applyHollowShardGuard,
|
||||
buildPaidShardArgs,
|
||||
buildRunManifest,
|
||||
parseRunManifest,
|
||||
retriesForFiles,
|
||||
RETRY_OVERRIDES,
|
||||
summarize,
|
||||
summaryExitCode,
|
||||
verifySliceResults,
|
||||
type PaidRunManifest,
|
||||
type ShardOutcome,
|
||||
type SliceResult,
|
||||
} from '../scripts/test-paid-shards';
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
|
||||
const outcome = (over: Partial<ShardOutcome>): ShardOutcome => ({
|
||||
shard: 1,
|
||||
files: ['test/skill-e2e-x.test.ts'],
|
||||
status: 'passed',
|
||||
exitCode: 0,
|
||||
elapsedMs: 1000,
|
||||
groupPid: null,
|
||||
executedTests: 3,
|
||||
...over,
|
||||
});
|
||||
|
||||
describe('run manifest (planner)', () => {
|
||||
test('live build: every paid file appears exactly once; planned slices partition 1..K', () => {
|
||||
const manifest = buildRunManifest({ tier: 'gate', sliceCount: 5, evalsAll: true, env: { EVALS_ALL: '1' } });
|
||||
const files = manifest.entries.map((e) => e.file);
|
||||
expect(new Set(files).size).toBe(files.length);
|
||||
const planned = manifest.entries.filter((e) => e.status === 'planned');
|
||||
expect(planned.length).toBeGreaterThan(20); // census sanity
|
||||
for (const entry of planned) {
|
||||
expect(entry.slice).toBeGreaterThanOrEqual(1);
|
||||
expect(entry.slice).toBeLessThanOrEqual(5);
|
||||
}
|
||||
// Round-robin balance: slice sizes differ by at most 1.
|
||||
const sizes = [1, 2, 3, 4, 5].map((i) => planned.filter((e) => e.slice === i).length);
|
||||
expect(Math.max(...sizes) - Math.min(...sizes)).toBeLessThanOrEqual(1);
|
||||
// Non-runnable entries carry slice 0 and a reason.
|
||||
for (const entry of manifest.entries.filter((e) => e.status !== 'planned')) {
|
||||
expect(entry.slice).toBe(0);
|
||||
expect(entry.reason ?? '').not.toBe('');
|
||||
}
|
||||
});
|
||||
|
||||
test('deterministic for identical inputs', () => {
|
||||
const opts = { tier: 'periodic' as const, sliceCount: 4, evalsAll: true, env: { EVALS_ALL: '1' } };
|
||||
expect(buildRunManifest(opts)).toEqual(buildRunManifest(opts));
|
||||
});
|
||||
|
||||
test('parse round-trips and rejects malformed manifests', () => {
|
||||
// EVALS_ALL short-circuits diff selection BEFORE any git walk: selection
|
||||
// is deliberately fail-closed on git errors, and CI's shallow free-tests
|
||||
// checkout has no base ref (first CI run failed here with
|
||||
// "ambiguous argument 'main...HEAD'").
|
||||
const manifest = buildRunManifest({ tier: 'gate', sliceCount: 2, evalsAll: false, env: { EVALS_ALL: '1' } });
|
||||
expect(parseRunManifest(JSON.stringify(manifest))).toEqual(manifest);
|
||||
expect(() => parseRunManifest('{}')).toThrow(/version/);
|
||||
expect(() => parseRunManifest(JSON.stringify({ ...manifest, tier: 'e2e' }))).toThrow(/tier/);
|
||||
expect(() => parseRunManifest(JSON.stringify({ ...manifest, sliceCount: 0 }))).toThrow(/sliceCount/);
|
||||
const outOfRange = {
|
||||
...manifest,
|
||||
entries: [{ file: 'test/skill-e2e-x.test.ts', slice: 9, status: 'planned' }],
|
||||
};
|
||||
expect(() => parseRunManifest(JSON.stringify(outOfRange))).toThrow(/out-of-range/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('slice-result reconciliation (report)', () => {
|
||||
const manifest: PaidRunManifest = {
|
||||
version: 1,
|
||||
tier: 'gate',
|
||||
evalsAll: false,
|
||||
sliceCount: 2,
|
||||
selectionReason: 'test fixture',
|
||||
entries: [
|
||||
{ file: 'test/skill-e2e-a.test.ts', slice: 1, status: 'planned' },
|
||||
{ file: 'test/skill-e2e-b.test.ts', slice: 2, status: 'planned' },
|
||||
{ file: 'test/skill-e2e-c.test.ts', slice: 0, status: 'skipped-by-diff', reason: 'unselected' },
|
||||
],
|
||||
};
|
||||
const slice = (index: number, files: string[], status: ShardOutcome['status'] = 'passed'): SliceResult => ({
|
||||
version: 1,
|
||||
tier: 'gate',
|
||||
sliceIndex: index,
|
||||
sliceCount: 2,
|
||||
outcomes: files.map((f) => ({ files: [f], status, exitCode: 0, elapsedMs: 5, executedTests: 2 })),
|
||||
});
|
||||
|
||||
test('all slices present and passing → ok', () => {
|
||||
const verdict = verifySliceResults(manifest, [
|
||||
slice(1, ['test/skill-e2e-a.test.ts']),
|
||||
slice(2, ['test/skill-e2e-b.test.ts']),
|
||||
]);
|
||||
expect(verdict).toEqual({ ok: true, problems: [] });
|
||||
});
|
||||
|
||||
test('a missing slice artifact is a FAILURE, not an absence', () => {
|
||||
const verdict = verifySliceResults(manifest, [slice(1, ['test/skill-e2e-a.test.ts'])]);
|
||||
expect(verdict.ok).toBe(false);
|
||||
expect(verdict.problems.join('\n')).toContain('slice 2/2 reported NO result');
|
||||
});
|
||||
|
||||
test('a planned shard nobody reported fails even when its slice reported', () => {
|
||||
const verdict = verifySliceResults(manifest, [
|
||||
slice(1, []),
|
||||
slice(2, ['test/skill-e2e-b.test.ts']),
|
||||
]);
|
||||
expect(verdict.ok).toBe(false);
|
||||
expect(verdict.problems.join('\n')).toContain('never reported');
|
||||
});
|
||||
|
||||
test('wrong-slice, duplicate, cross-tier, and failing outcomes all surface', () => {
|
||||
const wrongSlice = verifySliceResults(manifest, [
|
||||
slice(1, ['test/skill-e2e-b.test.ts']),
|
||||
slice(2, ['test/skill-e2e-a.test.ts']),
|
||||
]);
|
||||
expect(wrongSlice.ok).toBe(false);
|
||||
const failing = verifySliceResults(manifest, [
|
||||
slice(1, ['test/skill-e2e-a.test.ts'], 'failed'),
|
||||
slice(2, ['test/skill-e2e-b.test.ts']),
|
||||
]);
|
||||
expect(failing.problems.join('\n')).toContain('test/skill-e2e-a.test.ts: failed');
|
||||
const crossTier = verifySliceResults(manifest, [
|
||||
{ ...slice(1, ['test/skill-e2e-a.test.ts']), tier: 'periodic' },
|
||||
slice(2, ['test/skill-e2e-b.test.ts']),
|
||||
]);
|
||||
expect(crossTier.problems.join('\n')).toContain('ran tier periodic');
|
||||
});
|
||||
});
|
||||
|
||||
describe('hollow-shard guard', () => {
|
||||
test('EVALS_ALL: passed with 0 executed tests becomes passed-empty and fails the run', () => {
|
||||
const guarded = applyHollowShardGuard([outcome({ executedTests: 0 })], { evalsAll: true, warn: () => {} });
|
||||
expect(guarded[0].status).toBe('passed-empty');
|
||||
const summary = summarize(guarded);
|
||||
expect(summary.failed).toBe(1);
|
||||
expect(summaryExitCode(summary)).toBe(1);
|
||||
});
|
||||
|
||||
test('selective run: same shape stays passed, warns once', () => {
|
||||
const warnings: string[] = [];
|
||||
const guarded = applyHollowShardGuard([outcome({ executedTests: 0 })], {
|
||||
evalsAll: false, warn: (line) => warnings.push(line),
|
||||
});
|
||||
expect(guarded[0].status).toBe('passed');
|
||||
expect(warnings).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('unknown executedTests (null) is never guessed hollow', () => {
|
||||
const guarded = applyHollowShardGuard([outcome({ executedTests: null })], { evalsAll: true });
|
||||
expect(guarded[0].status).toBe('passed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('retry parity', () => {
|
||||
test('overrides exist only for the files whose matrix rows earned them, and each names a real file', () => {
|
||||
expect(Object.keys(RETRY_OVERRIDES).sort()).toEqual([
|
||||
'test/skill-e2e-office-hours-auto-mode.test.ts',
|
||||
'test/skill-e2e-plan-mode-no-op.test.ts',
|
||||
'test/skill-e2e-workflow.test.ts',
|
||||
]);
|
||||
for (const file of Object.keys(RETRY_OVERRIDES)) {
|
||||
expect(fs.existsSync(path.join(ROOT, file)), `stale RETRY_OVERRIDES entry: ${file}`).toBe(true);
|
||||
}
|
||||
expect(retriesForFiles(['test/skill-e2e-workflow.test.ts'])).toBe(2);
|
||||
expect(retriesForFiles(['test/skill-e2e-retro.test.ts'])).toBe(1);
|
||||
expect(buildPaidShardArgs(['x'], 1000, 4, 2)).toContain('2');
|
||||
expect(buildPaidShardArgs(['x'], 1000, 4).join(' ')).toContain('--retry 1');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* Parent/child selection-drift pins for EVALS_SELECTION_JSON.
|
||||
*
|
||||
* The sharded paid runner computes the diff selection ONCE in the parent
|
||||
* (computePaidDiffSelection in scripts/test-paid-shards.ts), serializes it
|
||||
* (serializePaidDiffSelection) into every shard child's env, and
|
||||
* test/helpers/e2e-helpers.ts adopts it at module load (parseEvalsSelectionJson
|
||||
* via resolveModuleSelection) instead of re-deriving it per shard — which,
|
||||
* whenever touchfiles-data.ts was in the diff, spawned one bun subprocess PER
|
||||
* CHILD to evaluate the old data file (test-selection.ts map-diff path).
|
||||
*
|
||||
* These pins hold the two sides to IDENTICAL selection decisions across the
|
||||
* serialize/parse boundary, and the child to fail-open (local recompute with
|
||||
* one stderr warning) on any parse/shape failure.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
computePaidDiffSelection,
|
||||
serializePaidDiffSelection,
|
||||
type PaidDiffSelection,
|
||||
} from '../scripts/test-paid-shards';
|
||||
import { parseEvalsSelectionJson, resolveModuleSelection } from './helpers/e2e-helpers';
|
||||
|
||||
/** The parent's per-test decision shape (PaidDiffSelection.selectedNames). */
|
||||
const parentWouldRun = (selection: PaidDiffSelection, name: string): boolean =>
|
||||
selection.selectedNames === null || selection.selectedNames.has(name);
|
||||
|
||||
/** The child's per-test decision shape (testIfSelected / describeIfSelected). */
|
||||
const childWouldRun = (selected: string[] | null, name: string): boolean =>
|
||||
selected === null || selected.includes(name);
|
||||
|
||||
const NAMES = ['qa-workflow', 'review-army', 'ship-docsync', 'unmapped-test'];
|
||||
|
||||
describe('EVALS_SELECTION_JSON parent -> child propagation', () => {
|
||||
test('a concrete selection round-trips to identical decisions', () => {
|
||||
const fixture: PaidDiffSelection = {
|
||||
selectedNames: new Set(['qa-workflow', 'ship-docsync']),
|
||||
reason: 'diff',
|
||||
totalTests: 4,
|
||||
};
|
||||
const parsed = parseEvalsSelectionJson(serializePaidDiffSelection(fixture));
|
||||
expect(parsed.selected).toEqual(['qa-workflow', 'ship-docsync']);
|
||||
expect(parsed.reason).toBe('diff');
|
||||
for (const name of NAMES) {
|
||||
expect(childWouldRun(parsed.selected, name), name).toBe(parentWouldRun(fixture, name));
|
||||
}
|
||||
});
|
||||
|
||||
test('run-all (null) round-trips to null — child runs everything', () => {
|
||||
// computePaidDiffSelection is the REAL parent function; EVALS_ALL is its
|
||||
// git-free path, so the serializer sees input exactly as produced.
|
||||
const selection = computePaidDiffSelection({ EVALS_ALL: '1' } as NodeJS.ProcessEnv);
|
||||
expect(selection.selectedNames).toBeNull();
|
||||
const parsed = parseEvalsSelectionJson(serializePaidDiffSelection(selection));
|
||||
expect(parsed.selected).toBeNull();
|
||||
for (const name of NAMES) {
|
||||
expect(childWouldRun(parsed.selected, name)).toBe(parentWouldRun(selection, name));
|
||||
}
|
||||
});
|
||||
|
||||
test('empty selection stays empty — nothing selected is NOT run-all', () => {
|
||||
const fixture: PaidDiffSelection = { selectedNames: new Set(), reason: 'diff', totalTests: 4 };
|
||||
const parsed = parseEvalsSelectionJson(serializePaidDiffSelection(fixture));
|
||||
expect(parsed.selected).toEqual([]);
|
||||
for (const name of NAMES) {
|
||||
expect(childWouldRun(parsed.selected, name)).toBe(false);
|
||||
expect(parentWouldRun(fixture, name)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('parser THROWS on malformed JSON and wrong shapes', () => {
|
||||
expect(() => parseEvalsSelectionJson('{"selected": ')).toThrow();
|
||||
expect(() => parseEvalsSelectionJson('null')).toThrow();
|
||||
expect(() => parseEvalsSelectionJson('[1,2]')).toThrow();
|
||||
expect(() => parseEvalsSelectionJson('{"selected": 42}')).toThrow();
|
||||
expect(() => parseEvalsSelectionJson('{"selected": ["a", 7]}')).toThrow();
|
||||
});
|
||||
|
||||
test('malformed EVALS_SELECTION_JSON falls back to local compute with one stderr warning', () => {
|
||||
const warnings: string[] = [];
|
||||
let computed = 0;
|
||||
const result = resolveModuleSelection(
|
||||
'{"selected": 42}',
|
||||
() => { computed += 1; return ['locally-computed']; },
|
||||
(text) => warnings.push(text),
|
||||
);
|
||||
expect(result).toEqual(['locally-computed']); // fail-open preserved
|
||||
expect(computed).toBe(1);
|
||||
expect(warnings.length).toBe(1);
|
||||
expect(warnings[0]).toContain('EVALS_SELECTION_JSON');
|
||||
});
|
||||
|
||||
test('absent env var computes locally, silently (non-sharded entrypoints unchanged)', () => {
|
||||
const writes: string[] = [];
|
||||
let computed = 0;
|
||||
const result = resolveModuleSelection(
|
||||
undefined,
|
||||
() => { computed += 1; return null; },
|
||||
(text) => writes.push(text),
|
||||
);
|
||||
expect(result).toBeNull();
|
||||
expect(computed).toBe(1);
|
||||
expect(writes.length).toBe(0);
|
||||
});
|
||||
|
||||
test('a valid env var short-circuits local derivation entirely', () => {
|
||||
let computed = 0;
|
||||
const result = resolveModuleSelection(
|
||||
serializePaidDiffSelection({ selectedNames: new Set(['a']), reason: 'diff', totalTests: 1 }),
|
||||
() => { computed += 1; return null; },
|
||||
() => {},
|
||||
);
|
||||
expect(result).toEqual(['a']);
|
||||
expect(computed).toBe(0); // no git walk, no map-diff bun subprocess
|
||||
});
|
||||
});
|
||||
@@ -11,6 +11,7 @@
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
@@ -43,15 +44,22 @@ describe('paid test enumeration', () => {
|
||||
// kept here as a regression pin: its glob-invisibility is exactly how
|
||||
// two gate tests went unexecuted for ~8 releases before the rehoming.
|
||||
expect(isPaidTestFile('test/skill-e2e.test.ts')).toBe(false);
|
||||
expect(isPaidTestFile('test/codex-e2e-recommendation-substance.test.ts')).toBe(false);
|
||||
expect(isPaidTestFile('test/paid-shards.test.ts')).toBe(false);
|
||||
// The 2026-08 orphan fix: these four were API-spending files OUTSIDE the
|
||||
// globs — self-skipping in the free suite and absent from the paid
|
||||
// census, so they could never run in any lane.
|
||||
expect(isPaidTestFile('test/codex-e2e-recommendation-substance.test.ts')).toBe(true);
|
||||
expect(isPaidTestFile('test/codex-e2e-plan-format.test.ts')).toBe(true);
|
||||
expect(isPaidTestFile('test/llm-judge-recommendation.test.ts')).toBe(true);
|
||||
expect(isPaidTestFile('test/carve-section-loading.test.ts')).toBe(true);
|
||||
expect(isPaidTestFile('test/skill-llm-eval-spec.test.ts')).toBe(true);
|
||||
});
|
||||
|
||||
test('discovers files and gives each one its own shard', () => {
|
||||
const files = collectPaidTestFiles();
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
expect(files.every(isPaidTestFile)).toBe(true);
|
||||
expect(PAID_TEST_GLOBS.length).toBe(6);
|
||||
expect(PAID_TEST_GLOBS.length).toBe(7);
|
||||
|
||||
const shards = planPaidShards(files);
|
||||
expect(shards.flat().sort()).toEqual([...files].sort());
|
||||
@@ -108,10 +116,17 @@ describe('tier classification', () => {
|
||||
describe('shard execution', () => {
|
||||
const BUSY_LOOP = 'const end = Date.now() + 600000; while (Date.now() < end) {}';
|
||||
|
||||
// PIN UPDATE (deliberate): the strict expectedFiles check is now enforced
|
||||
// for injected fake commands too (drift fix toward the free runner's
|
||||
// behavior), so a fake PASSING command must print a synthetic bun terminal
|
||||
// summary — a summary-less exit 0 is the truncation class and reads FAILED.
|
||||
const PASS_WITH_SUMMARY = 'console.log("ok"); console.log("Ran 1 tests across 1 files. [1ms]")';
|
||||
|
||||
const commandFor = (files: string[]) => {
|
||||
if (files[0] === 'spin') return { command: process.execPath, args: ['-e', BUSY_LOOP] };
|
||||
if (files[0] === 'fail') return { command: process.execPath, args: ['-e', 'process.exit(3)'] };
|
||||
return { command: process.execPath, args: ['-e', 'console.log("ok")'] };
|
||||
if (files[0] === 'silent-pass') return { command: process.execPath, args: ['-e', 'console.log("ok")'] };
|
||||
return { command: process.execPath, args: ['-e', PASS_WITH_SUMMARY] };
|
||||
};
|
||||
|
||||
test('a spinning shard times out, is killed, and the run continues', async () => {
|
||||
@@ -146,6 +161,48 @@ describe('shard execution', () => {
|
||||
expect(lines.some((l) => /PASSED in \d+s/.test(l))).toBe(true);
|
||||
}, 30_000);
|
||||
|
||||
test('exit 0 WITHOUT the terminal summary is FAILED — enforced for injected commands too', async () => {
|
||||
// The invisible-non-execution backstop: previously the paid runner
|
||||
// exempted injected commandFor from the expectedFiles check, so a fake
|
||||
// that exited 0 without bun's terminal summary recorded 'passed'. Now it
|
||||
// matches the free runner: enforcement always on.
|
||||
const summary = await runPaidShards([['silent-pass']], {
|
||||
timeoutMs: 30_000, jobs: 1, commandFor, log: () => {},
|
||||
});
|
||||
expect(summary.outcomes[0].status).toBe('failed');
|
||||
}, 30_000);
|
||||
|
||||
test('shard output spools to a per-shard log file; failures name the path', async () => {
|
||||
const logDir = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-shard-logs-'));
|
||||
const lines: string[] = [];
|
||||
try {
|
||||
const summary = await runPaidShards([['fail'], ['pass']], {
|
||||
timeoutMs: 30_000, jobs: 2, commandFor, logDir, log: (line) => lines.push(line),
|
||||
});
|
||||
const byName = (name: string) => summary.outcomes.find((o) => o.files[0] === name) as ShardOutcome;
|
||||
expect(byName('fail').status).toBe('failed');
|
||||
expect(byName('pass').status).toBe('passed');
|
||||
|
||||
// One log per shard, named by slug, and it holds the child's full stream
|
||||
// (nothing buffered in RAM: the file IS the record).
|
||||
const logs = fs.readdirSync(logDir).sort();
|
||||
expect(logs.length).toBe(2);
|
||||
expect(logs.some((f) => f.includes('fail'))).toBe(true);
|
||||
const passLog = logs.find((f) => f.includes('pass')) as string;
|
||||
expect(fs.readFileSync(path.join(logDir, passLog), 'utf8')).toContain('Ran 1 tests across 1 files.');
|
||||
|
||||
// Every shard announces its log path up front; the FAILED terminal line
|
||||
// repeats it, the PASSED one stays clean.
|
||||
expect(lines.filter((l) => l.includes('full log:') && !l.includes('FAILED')).length).toBe(2);
|
||||
const failLine = lines.find((l) => l.includes('FAILED')) as string;
|
||||
expect(failLine).toContain(logDir);
|
||||
const passLine = lines.find((l) => l.includes('PASSED')) as string;
|
||||
expect(passLine).not.toContain(logDir);
|
||||
} finally {
|
||||
fs.rmSync(logDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
test('summarize reports shards that never ran', () => {
|
||||
const summary = summarize([
|
||||
{ shard: 1, files: ['a'], status: 'passed', exitCode: 0, elapsedMs: 1, groupPid: 1 },
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* The periodic exclude list is a set of DECISIONS, not a place tests go to
|
||||
* die: every entry names a real file (a deleted/renamed file must drop its
|
||||
* entry) and carries a non-empty reason + tracking pointer (the re-entry
|
||||
* condition lives there). The runner surfaces each exclusion per run, and
|
||||
* removing an entry re-activates the file on the next weekly lane.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
import { PERIODIC_CI_EXCLUDE } from './helpers/periodic-exclude-data';
|
||||
import { isPaidTestFile } from './helpers/paid-test-set';
|
||||
import { selectPaidTestFiles } from '../scripts/test-paid-shards';
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
|
||||
describe('periodic exclude policy', () => {
|
||||
test('every entry names a real paid file and carries reason + tracking', () => {
|
||||
const entries = Object.entries(PERIODIC_CI_EXCLUDE);
|
||||
expect(entries.length).toBeGreaterThan(0);
|
||||
for (const [file, meta] of entries) {
|
||||
expect(fs.existsSync(path.join(ROOT, file)), `stale exclude entry: ${file}`).toBe(true);
|
||||
expect(isPaidTestFile(file), `${file} is not a paid file — exclusion is meaningless`).toBe(true);
|
||||
expect(meta.reason.length, `${file}: empty reason`).toBeGreaterThan(20);
|
||||
expect(meta.tracking.length, `${file}: empty tracking pointer`).toBeGreaterThan(5);
|
||||
}
|
||||
});
|
||||
|
||||
test('exclusions apply to the periodic tier only, with the reason surfaced', () => {
|
||||
const files = Object.keys(PERIODIC_CI_EXCLUDE);
|
||||
const periodic = selectPaidTestFiles(files, 'periodic');
|
||||
expect(periodic.selected).toEqual([]);
|
||||
for (const { reason } of periodic.excluded) {
|
||||
expect(reason).toStartWith('excluded: ');
|
||||
expect(reason).toContain('[');
|
||||
}
|
||||
// Gate tier ignores the list (these files are periodic-tier anyway; the
|
||||
// list must never leak into gate semantics).
|
||||
const gate = selectPaidTestFiles(files, 'gate');
|
||||
for (const { reason } of gate.excluded) {
|
||||
expect(reason).not.toStartWith('excluded: ');
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -182,6 +182,35 @@ describe('gstack-wtree', () => {
|
||||
});
|
||||
});
|
||||
|
||||
test('racy-git window: a same-size rewrite pinned to the index timestamp changes the fingerprint', () => {
|
||||
withScratchRepo((repoDir, wtree) => {
|
||||
const file = path.join(repoDir, 'a.txt');
|
||||
const indexPath = path.join(repoDir, '.git', 'index');
|
||||
// ctime can't be restored after a rewrite; production hits this window
|
||||
// when everything lands in the same second (ctime SECONDS match).
|
||||
// trustctime=false isolates the racy mechanism deterministically
|
||||
// instead of racing a second boundary.
|
||||
gitIn(repoDir, 'config core.trustctime false');
|
||||
// Pin the cached entry's mtime to a fixed timestamp (zero nsec, so the
|
||||
// restore below is exact even on USE_NSEC git builds).
|
||||
const pinned = new Date('2026-01-01T12:00:00Z');
|
||||
fs.utimesSync(file, pinned, pinned);
|
||||
gitIn(repoDir, 'add a.txt');
|
||||
const clean = wtree();
|
||||
// Same-size rewrite restored to the pinned stat, with the index file
|
||||
// itself pinned to the SAME timestamp: the entry is stat-identical to
|
||||
// its stale cache and sits exactly on git's racy-git boundary.
|
||||
// gstack-wtree must carry the real index's mtime onto its temp copy —
|
||||
// a fresh-stamped copy marks the entry non-racy, trusts the stale stat
|
||||
// cache, and the edit vanishes from the fingerprint (evidence would
|
||||
// stay FRESH after a source change).
|
||||
fs.writeFileSync(file, 'howdy\n'); // same byte length as 'hello\n'
|
||||
fs.utimesSync(file, pinned, pinned);
|
||||
fs.utimesSync(indexPath, pinned, pinned);
|
||||
expect(wtree()).not.toBe(clean);
|
||||
});
|
||||
});
|
||||
|
||||
test('exits non-zero outside a git repo', () => {
|
||||
const nonGit = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-wtree-nongit-'));
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
/**
|
||||
* Direct pins for runShardChild (scripts/test-strict-output.ts) — the shared
|
||||
* spawn/detached/group-kill/wall-timer/reap lifecycle extracted from the paid
|
||||
* runner's runPaidShard, designed for scripts/test-free-shards.ts to migrate
|
||||
* onto next. test/paid-shards.test.ts pins the paid runner end-to-end; these
|
||||
* pin the helper's own contract so the free-runner migration has a floor.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import type { ChildProcess } from 'child_process';
|
||||
import { runShardChild } from '../scripts/test-strict-output';
|
||||
|
||||
/** Collect the child's full stdout+stderr, resolving only when drained. */
|
||||
function collectingHook(chunks: string[]) {
|
||||
return (child: ChildProcess): Array<Promise<void>> => {
|
||||
const consume = (stream: NodeJS.ReadableStream | null): Promise<void> =>
|
||||
stream
|
||||
? new Promise((resolve, reject) => {
|
||||
stream.on('data', (chunk: Buffer | string) => chunks.push(chunk.toString()));
|
||||
stream.on('end', resolve);
|
||||
stream.on('error', reject);
|
||||
})
|
||||
: Promise.resolve();
|
||||
return [consume(child.stdout), consume(child.stderr)];
|
||||
};
|
||||
}
|
||||
|
||||
describe('runShardChild', () => {
|
||||
test('clean exit: exitCode 0, not timed out, output drained before resolve', async () => {
|
||||
const chunks: string[] = [];
|
||||
const result = await runShardChild({
|
||||
command: process.execPath,
|
||||
args: ['-e', 'console.log("hello-from-child")'],
|
||||
cwd: process.cwd(),
|
||||
env: process.env,
|
||||
timeoutMs: 30_000,
|
||||
hookStreams: collectingHook(chunks),
|
||||
});
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.timedOut).toBe(false);
|
||||
expect(result.groupPid).toBeGreaterThan(0);
|
||||
// The hookStreams promises are awaited AFTER close — trailing output is
|
||||
// fully drained before callers read their classifier/log state.
|
||||
expect(chunks.join('')).toContain('hello-from-child');
|
||||
}, 30_000);
|
||||
|
||||
test('non-zero exit code propagates untouched', async () => {
|
||||
const result = await runShardChild({
|
||||
command: process.execPath,
|
||||
args: ['-e', 'process.exit(7)'],
|
||||
cwd: process.cwd(),
|
||||
env: process.env,
|
||||
timeoutMs: 30_000,
|
||||
hookStreams: () => [],
|
||||
});
|
||||
expect(result.exitCode).toBe(7);
|
||||
expect(result.timedOut).toBe(false);
|
||||
}, 30_000);
|
||||
|
||||
test('a spinning child is group-SIGKILLed at the wall deadline and reported timedOut', async () => {
|
||||
const startedAt = Date.now();
|
||||
const result = await runShardChild({
|
||||
command: process.execPath,
|
||||
// A real busy loop: an in-process timer could never fire in this child.
|
||||
args: ['-e', 'const end = Date.now() + 600000; while (Date.now() < end) {}'],
|
||||
cwd: process.cwd(),
|
||||
env: process.env,
|
||||
timeoutMs: 1_200,
|
||||
hookStreams: () => [],
|
||||
});
|
||||
expect(result.timedOut).toBe(true);
|
||||
expect(Date.now() - startedAt).toBeLessThan(30_000);
|
||||
if (process.platform !== 'win32') {
|
||||
// The whole group is gone, not left to burn a core.
|
||||
expect(() => process.kill(result.groupPid as number, 0)).toThrow();
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
test('a spawn failure THROWS so callers keep their could-not-run handling', async () => {
|
||||
await expect(runShardChild({
|
||||
command: path.join(os.tmpdir(), 'definitely-not-a-real-binary-8b1f'),
|
||||
args: [],
|
||||
cwd: process.cwd(),
|
||||
env: process.env,
|
||||
timeoutMs: 5_000,
|
||||
hookStreams: () => [],
|
||||
})).rejects.toThrow();
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -23,6 +23,7 @@
|
||||
* A/B and matrix evals (test/helpers/auq-sdk-capture.ts).
|
||||
*/
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
@@ -86,6 +87,6 @@ describeE2E('AskUserQuestion format compliance (gate)', () => {
|
||||
);
|
||||
}
|
||||
},
|
||||
300_000,
|
||||
CAPTURE_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
* (N SDK runs, ~$0.50-1 each).
|
||||
*/
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
@@ -99,6 +100,6 @@ describeE2E('AUQ consistency across runs (periodic)', () => {
|
||||
`format elements every run; substance ${minSub}-${maxSub}`,
|
||||
);
|
||||
},
|
||||
N_RUNS * 300_000 + 60_000,
|
||||
N_RUNS * CAPTURE_MS + 60_000,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
* Run a subset in the foreground with AUQ_MATRIX_ONLY="plan-eng-review,cso".
|
||||
*/
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
@@ -174,7 +175,7 @@ describeE2E('AUQ behavioral matrix (periodic)', () => {
|
||||
);
|
||||
}
|
||||
},
|
||||
300_000,
|
||||
CAPTURE_MS,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
* strictly less unrelated review-section text in context.
|
||||
*/
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
@@ -109,6 +110,6 @@ describeE2E('AUQ no-degradation: verbose vs carved (periodic)', () => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('[AUQ-AB] NO DEGRADATION:\n' + summary);
|
||||
},
|
||||
600_000,
|
||||
CAPTURE_LONG_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -38,6 +38,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
|
||||
import * as fs from 'fs';
|
||||
@@ -67,6 +68,8 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
|
||||
// claude would resolve). The preference file path keys on this slug.
|
||||
const slugBin = path.join(ROOT, 'bin', 'gstack-slug');
|
||||
const slugRes = spawnSync(slugBin, [], {
|
||||
// LIVE-REPO CWD: gstack-slug resolves the slug from this repo's git
|
||||
// remote — must match what the spawned claude (repo cwd) resolves.
|
||||
cwd: ROOT,
|
||||
env: { ...process.env, GSTACK_HOME: tmpHome },
|
||||
encoding: 'utf-8',
|
||||
@@ -111,7 +114,7 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
|
||||
skillName: 'plan-ceo-review',
|
||||
inPlanMode: true,
|
||||
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
|
||||
timeoutMs: 540_000,
|
||||
timeoutMs: CAPTURE_LONG_MS,
|
||||
env: { GSTACK_HOME: tmpHome, CONDUCTOR_WORKSPACE_PATH: tmpHome },
|
||||
});
|
||||
|
||||
@@ -135,5 +138,5 @@ describeE2E('AUTO_DECIDE opt-in preserved under Conductor flags (periodic)', ()
|
||||
} finally {
|
||||
try { fs.rmSync(tmpHome, { recursive: true, force: true }); } catch { /* best-effort */ }
|
||||
}
|
||||
}, 660_000);
|
||||
}, PTY_MS);
|
||||
});
|
||||
|
||||
@@ -26,6 +26,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { PTY_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
@@ -184,6 +185,6 @@ describeE2E('/autoplan chain ordering (periodic)', () => {
|
||||
try { fs.rmSync(tempDir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
},
|
||||
1_200_000, // 20 min absolute test ceiling
|
||||
PTY_LONG_MS, // 20 min absolute test ceiling
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId, evalsEnabled,
|
||||
@@ -98,7 +99,7 @@ Add a new /greet skill that prints a welcome message.
|
||||
testName: 'autoplan-dual-voice',
|
||||
workingDirectory: workDir,
|
||||
prompt: `/autoplan ${planPath}`,
|
||||
timeout: 600_000, // 10 min
|
||||
timeout: CAPTURE_LONG_MS, // 10 min
|
||||
// /autoplan spawns subagents and calls codex via Bash; it needs the
|
||||
// full tool set to get past Phase 1. Bash+Read+Write alone wasn't
|
||||
// enough — the skill stalled trying to invoke Agent/Skill.
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { ClaudeAdapter } from './helpers/providers/claude';
|
||||
import { GptAdapter } from './helpers/providers/gpt';
|
||||
import { GeminiAdapter } from './helpers/providers/gemini';
|
||||
@@ -94,7 +95,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
process.stderr.write(`\nclaude live smoke: SKIPPED — ${check.reason}\n`);
|
||||
return;
|
||||
}
|
||||
const result = await claude.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 });
|
||||
const result = await claude.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS });
|
||||
if (result.error) {
|
||||
throw new Error(`claude errored: ${result.error.code} — ${result.error.reason}`);
|
||||
}
|
||||
@@ -106,7 +107,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
expect(result.modelUsed.length).toBeGreaterThan(0);
|
||||
const cost = claude.estimateCost(result.tokens, result.modelUsed);
|
||||
expect(cost).toBeGreaterThan(0);
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
test('gpt: trivial prompt produces parseable output', async () => {
|
||||
const check = await gpt.available();
|
||||
@@ -114,7 +115,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
process.stderr.write(`\ngpt live smoke: SKIPPED — ${check.reason}\n`);
|
||||
return;
|
||||
}
|
||||
const result = await gpt.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 });
|
||||
const result = await gpt.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS });
|
||||
if (result.error) {
|
||||
throw new Error(`gpt errored: ${result.error.code} — ${result.error.reason}`);
|
||||
}
|
||||
@@ -125,7 +126,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
expect(typeof result.modelUsed).toBe('string');
|
||||
const cost = gpt.estimateCost(result.tokens, result.modelUsed);
|
||||
expect(cost).toBeGreaterThan(0);
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
test('gemini: trivial prompt produces parseable output', async () => {
|
||||
const check = await gemini.available();
|
||||
@@ -133,7 +134,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
process.stderr.write(`\ngemini live smoke: SKIPPED — ${check.reason}\n`);
|
||||
return;
|
||||
}
|
||||
const result = await gemini.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 });
|
||||
const result = await gemini.run({ prompt: PROMPT, workdir, timeoutMs: JUDGE_MS });
|
||||
if (result.error) {
|
||||
// auth / rate_limit are ENVIRONMENT conditions the test can't act on
|
||||
// (e.g. Google deprecated the individual code-assist auth path — the
|
||||
@@ -155,7 +156,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
expect(result.durationMs).toBeGreaterThan(0);
|
||||
expect(typeof result.modelUsed).toBe('string');
|
||||
expect(result.modelUsed.length).toBeGreaterThan(0);
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
test('timeout error surfaces as error.code=timeout (no exception)', async () => {
|
||||
// Use whatever adapter is available first — all three should share timeout semantics.
|
||||
@@ -183,7 +184,7 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
prompt: PROMPT,
|
||||
workdir,
|
||||
providers: ['claude', 'gpt', 'gemini'],
|
||||
timeoutMs: 120_000,
|
||||
timeoutMs: JUDGE_MS,
|
||||
skipUnavailable: false,
|
||||
});
|
||||
expect(report.entries).toHaveLength(3);
|
||||
@@ -201,5 +202,5 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
|
||||
if (!hadSuccess) {
|
||||
process.stderr.write('\nrunBenchmark live: no provider produced a clean result (no auth?)\n');
|
||||
}
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
@@ -150,7 +151,7 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
||||
fs.rmSync(fakeBinDir, { recursive: true, force: true });
|
||||
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
test('privacy gate does NOT fire when artifacts_sync_mode_prompted is already true', async () => {
|
||||
// Same staging, but prompted=true this time. Gate should be silent.
|
||||
@@ -228,5 +229,5 @@ describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
||||
fs.rmSync(fakeBinDir, { recursive: true, force: true });
|
||||
fs.rmSync(tempHome, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
+12
-11
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled,
|
||||
@@ -47,7 +48,7 @@ describeIfSelected('Skill E2E tests', [
|
||||
Report the results of each command.`,
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 7,
|
||||
timeout: 60_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'browse-basic',
|
||||
runId,
|
||||
});
|
||||
@@ -56,7 +57,7 @@ Report the results of each command.`,
|
||||
recordE2E(evalCollector, 'browse basic commands', 'Skill E2E tests', result);
|
||||
expect(result.browseErrors).toHaveLength(0);
|
||||
expect(result.exitReason).toBe('success');
|
||||
}, 90_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
testConcurrentIfSelected('browse-snapshot', async () => {
|
||||
const result = await runSkillTest({
|
||||
@@ -69,7 +70,7 @@ Report the results of each command.`,
|
||||
Report what each command returned.`,
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 9,
|
||||
timeout: 60_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'browse-snapshot',
|
||||
runId,
|
||||
});
|
||||
@@ -81,7 +82,7 @@ Report what each command returned.`,
|
||||
console.warn('Browse errors (non-fatal):', result.browseErrors);
|
||||
}
|
||||
expect(result.exitReason).toBe('success');
|
||||
}, 90_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
testConcurrentIfSelected('skillmd-setup-discovery', async () => {
|
||||
// P2 (v1.2.0): the browse SETUP/binary-discovery block moved from the root
|
||||
@@ -104,7 +105,7 @@ Then run: $B text
|
||||
Report whether it worked.`,
|
||||
workingDirectory: tmpDir,
|
||||
maxTurns: 10,
|
||||
timeout: 60_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'skillmd-setup-discovery',
|
||||
runId,
|
||||
});
|
||||
@@ -112,7 +113,7 @@ Report whether it worked.`,
|
||||
recordE2E(evalCollector, 'SKILL.md setup block discovery', 'Skill E2E tests', result);
|
||||
expect(result.browseErrors).toHaveLength(0);
|
||||
expect(result.exitReason).toBe('success');
|
||||
}, 90_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
testConcurrentIfSelected('skillmd-no-local-binary', async () => {
|
||||
// Create a tmpdir with no browse binary — no local .claude/skills/gstack/browse/dist/browse
|
||||
@@ -149,7 +150,7 @@ Report the exact output. Do NOT try to fix or install anything — just report w
|
||||
|
||||
// Clean up
|
||||
try { fs.rmSync(emptyDir, { recursive: true, force: true }); } catch {}
|
||||
}, 60_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
testConcurrentIfSelected('skillmd-outside-git', async () => {
|
||||
// Create a tmpdir outside any git repo
|
||||
@@ -182,7 +183,7 @@ Report the exact output — either "READY: <path>" or "NEEDS_SETUP".`,
|
||||
|
||||
// Clean up
|
||||
try { fs.rmSync(nonGitDir, { recursive: true, force: true }); } catch {}
|
||||
}, 60_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
testConcurrentIfSelected('operational-learning', async () => {
|
||||
const opDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-oplearn-'));
|
||||
@@ -286,7 +287,7 @@ Log the operational learning now. Then say what you logged.`,
|
||||
|
||||
// Clean up
|
||||
try { fs.rmSync(opDir, { recursive: true, force: true }); } catch {}
|
||||
}, 90_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
testConcurrentIfSelected('session-awareness', async () => {
|
||||
const sessionDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-session-'));
|
||||
@@ -353,7 +354,7 @@ Since this is non-interactive, DO NOT actually call AskUserQuestion. Instead, wr
|
||||
Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple windows and may not remember what this conversation is about. Re-ground them.`,
|
||||
workingDirectory: sessionDir,
|
||||
maxTurns: 8,
|
||||
timeout: 60_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'session-awareness',
|
||||
runId,
|
||||
});
|
||||
@@ -394,7 +395,7 @@ Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple wi
|
||||
|
||||
// Clean up
|
||||
try { fs.rmSync(sessionDir, { recursive: true, force: true }); } catch {}
|
||||
}, 90_000);
|
||||
}, JUDGE_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
|
||||
|
||||
@@ -46,7 +47,7 @@ describeE2E('Conductor renders decisions as prose (periodic)', () => {
|
||||
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
|
||||
env: { CONDUCTOR_WORKSPACE_PATH: '/tmp/conductor-prose-e2e' },
|
||||
initialPlanContent: FLAWED_PLAN,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
// The decision must reach the human as prose. 'silent_write' (wrote findings
|
||||
@@ -65,5 +66,5 @@ describeE2E('Conductor renders decisions as prose (periodic)', () => {
|
||||
}
|
||||
// A prose-rendered decision brief was observed at some point in the run.
|
||||
expect(obs.proseAUQEverObserved).toBe(true);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId, evalsEnabled,
|
||||
@@ -163,7 +164,7 @@ describeIfSelected('Context Skills E2E (live-fire)', [
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 12,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-save-routing',
|
||||
runId,
|
||||
});
|
||||
@@ -185,7 +186,7 @@ describeIfSelected('Context Skills E2E (live-fire)', [
|
||||
expect(routedToContextSave).toBe(true);
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// ── 2. Round-trip: save then restore in the same session ─────────────
|
||||
testConcurrentIfSelected('context-save-then-restore-roundtrip', async () => {
|
||||
@@ -205,7 +206,7 @@ Do NOT use AskUserQuestion.`,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 25,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'context-save-then-restore-roundtrip',
|
||||
runId,
|
||||
});
|
||||
@@ -232,7 +233,7 @@ Do NOT use AskUserQuestion.`,
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
expect(restoreMentionsTitle).toBe(true);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 240_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// ── 3. /context-restore <fragment> loads the matching save ───────────
|
||||
testConcurrentIfSelected('context-restore-fragment-match', async () => {
|
||||
@@ -255,7 +256,7 @@ Do NOT use AskUserQuestion.`,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 10,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-restore-fragment-match',
|
||||
runId,
|
||||
});
|
||||
@@ -279,7 +280,7 @@ Do NOT use AskUserQuestion.`,
|
||||
expect(loadedPayments).toBe(true);
|
||||
expect(didNotLoadOthers).toBe(true);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// ── 4. /context-restore with zero saves → graceful empty-state ───────
|
||||
testConcurrentIfSelected('context-restore-empty-state', async () => {
|
||||
@@ -294,7 +295,7 @@ Do NOT use AskUserQuestion.`,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 8,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-restore-empty-state',
|
||||
runId,
|
||||
});
|
||||
@@ -319,7 +320,7 @@ Do NOT use AskUserQuestion.`,
|
||||
expect(routedToRestore).toBe(true);
|
||||
expect(gracefulMessage).toBe(true);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// ── 5. /context-restore list redirects to /context-save list ─────────
|
||||
testConcurrentIfSelected('context-restore-list-delegates', async () => {
|
||||
@@ -334,7 +335,7 @@ Do NOT use AskUserQuestion.`,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 8,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-restore-list-delegates',
|
||||
runId,
|
||||
});
|
||||
@@ -357,7 +358,7 @@ Do NOT use AskUserQuestion.`,
|
||||
expect(routedToRestore).toBe(true);
|
||||
expect(mentionsSaveList).toBe(true);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// ── 6. Legacy compat: pre-rename save files still load ───────────────
|
||||
testConcurrentIfSelected('context-restore-legacy-compat', async () => {
|
||||
@@ -381,7 +382,7 @@ Do NOT use AskUserQuestion.`,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 8,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-restore-legacy-compat',
|
||||
runId,
|
||||
});
|
||||
@@ -414,7 +415,7 @@ Do NOT use AskUserQuestion.`,
|
||||
expect(routedToRestore).toBe(true);
|
||||
expect(loadedLegacy).toBe(true);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// ── 7. /context-save list: default filters to current branch ─────────
|
||||
testConcurrentIfSelected('context-save-list-current-branch', async () => {
|
||||
@@ -437,7 +438,7 @@ Do NOT use AskUserQuestion.`,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 10,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-save-list-current-branch',
|
||||
runId,
|
||||
});
|
||||
@@ -472,7 +473,7 @@ Do NOT use AskUserQuestion.`,
|
||||
expect(hidesAlpha).toBe(true);
|
||||
expect(hidesBeta).toBe(true);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// ── 8. /context-save list --all: shows every branch ──────────────────
|
||||
testConcurrentIfSelected('context-save-list-all-branches', async () => {
|
||||
@@ -494,7 +495,7 @@ Do NOT use AskUserQuestion.`,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 10,
|
||||
allowedTools: ['Skill', 'Bash', 'Read', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-save-list-all-branches',
|
||||
runId,
|
||||
});
|
||||
@@ -520,5 +521,5 @@ Do NOT use AskUserQuestion.`,
|
||||
expect(routed).toBe(true);
|
||||
expect(filesShown).toBe(3);
|
||||
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
@@ -78,7 +79,7 @@ Output the diagram directly.`,
|
||||
workingDirectory: reviewCoverageDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'review-coverage-audit',
|
||||
runId,
|
||||
});
|
||||
@@ -107,7 +108,7 @@ Output the diagram directly.`,
|
||||
// At minimum, the agent should have read the source and test files
|
||||
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
|
||||
expect(readCalls.length).toBeGreaterThan(0);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Plan Eng Review Coverage Audit E2E ---
|
||||
@@ -153,7 +154,7 @@ Output the diagram directly.`,
|
||||
workingDirectory: planCoverageDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'plan-eng-coverage-audit',
|
||||
runId,
|
||||
});
|
||||
@@ -182,7 +183,7 @@ Output the diagram directly.`,
|
||||
// At minimum, the agent should have read the source and test files
|
||||
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
|
||||
expect(readCalls.length).toBeGreaterThan(0);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId, evalsEnabled,
|
||||
@@ -75,7 +76,7 @@ IMPORTANT:
|
||||
workingDirectory: csoDir,
|
||||
maxTurns: 30,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob', 'Agent'],
|
||||
timeout: 300_000,
|
||||
timeout: CAPTURE_MS,
|
||||
});
|
||||
|
||||
logCost('cso', result);
|
||||
@@ -106,7 +107,7 @@ IMPORTANT:
|
||||
}
|
||||
|
||||
recordE2E(evalCollector, 'cso-full-audit', 'e2e-cso', result);
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
describeIfSelected('CSO v2 — diff mode', ['cso-diff-mode'], () => {
|
||||
@@ -181,7 +182,7 @@ IMPORTANT:
|
||||
).toBe(true);
|
||||
|
||||
recordE2E(evalCollector, 'cso-diff-mode', 'e2e-cso', result);
|
||||
}, 400_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
describeIfSelected('CSO v2 — infra scope', ['cso-infra-scope'], () => {
|
||||
@@ -245,7 +246,7 @@ IMPORTANT:
|
||||
workingDirectory: csoInfraDir,
|
||||
maxTurns: 30,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
});
|
||||
|
||||
logCost('cso', result);
|
||||
@@ -259,5 +260,5 @@ IMPORTANT:
|
||||
).toBe(true);
|
||||
|
||||
recordE2E(evalCollector, 'cso-infra-scope', 'e2e-cso', result);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled,
|
||||
@@ -67,7 +68,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`,
|
||||
workingDirectory: landDir,
|
||||
maxTurns: 20,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'land-and-deploy-workflow',
|
||||
runId,
|
||||
});
|
||||
@@ -85,7 +86,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`,
|
||||
|
||||
const reportDir = path.join(landDir, '.gstack', 'deploy-reports');
|
||||
expect(fs.existsSync(reportDir)).toBe(true);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Land-and-Deploy First-Run E2E ---
|
||||
@@ -148,7 +149,7 @@ Just demonstrate the first-run dry-run output.`,
|
||||
workingDirectory: firstRunDir,
|
||||
maxTurns: 20,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'land-and-deploy-first-run',
|
||||
runId,
|
||||
});
|
||||
@@ -167,7 +168,7 @@ Just demonstrate the first-run dry-run output.`,
|
||||
const reportContent = fs.readFileSync(path.join(reportDir, reportFiles[0]), 'utf-8');
|
||||
const hasPlatform = reportContent.toLowerCase().includes('fly') || reportContent.toLowerCase().includes('first-run-app');
|
||||
expect(hasPlatform).toBe(true);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Land-and-Deploy Review Gate E2E ---
|
||||
@@ -226,7 +227,7 @@ Show what the readiness gate output would look like.`,
|
||||
workingDirectory: reviewDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'land-and-deploy-review-gate',
|
||||
runId,
|
||||
});
|
||||
@@ -246,7 +247,7 @@ Show what the readiness gate output would look like.`,
|
||||
const hasReviewMention = reportContent.toLowerCase().includes('review') ||
|
||||
reportContent.toLowerCase().includes('not run');
|
||||
expect(hasReviewMention).toBe(true);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Canary skill E2E ---
|
||||
@@ -294,7 +295,7 @@ Just create the directory structure and report files showing the correct schema.
|
||||
workingDirectory: canaryDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'canary-workflow',
|
||||
runId,
|
||||
});
|
||||
@@ -307,7 +308,7 @@ Just create the directory structure and report files showing the correct schema.
|
||||
const reportDir = path.join(canaryDir, '.gstack', 'canary-reports');
|
||||
const files = fs.readdirSync(reportDir, { recursive: true }) as string[];
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Benchmark skill E2E ---
|
||||
@@ -357,7 +358,7 @@ Just create the files showing the correct schema and report format.`,
|
||||
workingDirectory: benchDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'benchmark-workflow',
|
||||
runId,
|
||||
});
|
||||
@@ -372,7 +373,7 @@ Just create the files showing the correct schema and report format.`,
|
||||
const files = fs.readdirSync(baselineDir);
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Setup-Deploy skill E2E ---
|
||||
@@ -418,7 +419,7 @@ Just detect the platform and write the config.`,
|
||||
workingDirectory: setupDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'setup-deploy-workflow',
|
||||
runId,
|
||||
});
|
||||
@@ -434,7 +435,7 @@ Just detect the platform and write the config.`,
|
||||
expect(content.toLowerCase()).toContain('fly');
|
||||
expect(content).toContain('my-cool-app');
|
||||
expect(content).toContain('Deploy Configuration');
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import { callJudge } from './helpers/llm-judge';
|
||||
import {
|
||||
@@ -113,7 +114,7 @@ Skip research — work from your design knowledge. Skip the font preview page. S
|
||||
Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
|
||||
workingDirectory: designDir,
|
||||
maxTurns: 20,
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'design-consultation-core',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -178,7 +179,7 @@ Write DESIGN.md and CLAUDE.md (or update it) in the working directory.`,
|
||||
const claude = fs.readFileSync(claudePath, 'utf-8');
|
||||
expect(claude.toLowerCase()).toContain('design.md');
|
||||
}
|
||||
}, 420_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
testConcurrentIfSelected('design-consultation-research', async () => {
|
||||
// Test WebSearch integration — research phase only, no DESIGN.md generation
|
||||
@@ -202,7 +203,7 @@ Do NOT generate a full DESIGN.md — just research notes.`,
|
||||
// queued past the budget under concurrent API load. 90s budgets cannot
|
||||
// absorb one slow first completion; 300s is the repo's standard floor
|
||||
// for CI SDK tests. Outer timeout below rises to 360s for headroom.
|
||||
timeout: 300_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'design-consultation-research',
|
||||
runId,
|
||||
});
|
||||
@@ -232,7 +233,7 @@ Do NOT generate a full DESIGN.md — just research notes.`,
|
||||
}
|
||||
|
||||
try { fs.rmSync(researchDir, { recursive: true, force: true }); } catch {}
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
testConcurrentIfSelected('design-consultation-existing', async () => {
|
||||
// Pre-create a minimal DESIGN.md (independent of core test)
|
||||
@@ -250,7 +251,7 @@ There is already a DESIGN.md in this repo. Update it with a complete design syst
|
||||
Skip research. Skip font preview. Skip any AskUserQuestion calls — this is non-interactive.`,
|
||||
workingDirectory: designDir,
|
||||
maxTurns: 20,
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'design-consultation-existing',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -279,7 +280,7 @@ Skip research. Skip font preview. Skip any AskUserQuestion calls — this is non
|
||||
expect(hasColor).toBe(true);
|
||||
expect(hasSpacing).toBe(true);
|
||||
}
|
||||
}, 420_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
testConcurrentIfSelected('design-consultation-preview', async () => {
|
||||
// Test preview HTML generation only — no DESIGN.md (covered by core test)
|
||||
@@ -302,7 +303,7 @@ Do NOT write DESIGN.md — only the preview HTML.`,
|
||||
maxTurns: 8,
|
||||
// 300s, not 90s: this is the test that failed 3x at 0 turns/$0.00/93s
|
||||
// on PR #2533 CI — see the research test's comment for the class.
|
||||
timeout: 300_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'design-consultation-preview',
|
||||
runId,
|
||||
});
|
||||
@@ -331,7 +332,7 @@ Do NOT write DESIGN.md — only the preview HTML.`,
|
||||
}
|
||||
|
||||
try { fs.rmSync(previewDir, { recursive: true, force: true }); } catch {}
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Plan Design Review E2E (plan-mode) ---
|
||||
@@ -398,7 +399,7 @@ Skip the preamble bash block. Skip any AskUserQuestion calls — this is non-int
|
||||
IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan review, not a live site audit. Just read the plan file, review it, and edit it to fix the gaps.`,
|
||||
workingDirectory: reviewDir,
|
||||
maxTurns: 15,
|
||||
timeout: 300_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-design-review-plan-mode',
|
||||
runId,
|
||||
});
|
||||
@@ -437,7 +438,7 @@ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan
|
||||
} finally {
|
||||
try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {}
|
||||
}
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
testConcurrentIfSelected('plan-design-review-no-ui-scope', async () => {
|
||||
const reviewDir = setupReviewDir();
|
||||
@@ -472,7 +473,7 @@ Skip the preamble bash block. Skip any AskUserQuestion calls — this is non-int
|
||||
IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan review, not a live site audit.`,
|
||||
workingDirectory: reviewDir,
|
||||
maxTurns: 10,
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-design-review-no-ui-scope',
|
||||
runId,
|
||||
});
|
||||
@@ -496,7 +497,7 @@ IMPORTANT: Do NOT try to browse any URLs or use a browse binary. This is a plan
|
||||
} finally {
|
||||
try { fs.rmSync(reviewDir, { recursive: true, force: true }); } catch {}
|
||||
}
|
||||
}, 240_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Design Review E2E (live-site audit + fix) ---
|
||||
@@ -602,7 +603,7 @@ Read design-review/SKILL.md for the design review + fix workflow.
|
||||
Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion calls — this is non-interactive. Fix up to 3 issues max. Write your report to ./design-audit.md.`,
|
||||
workingDirectory: qaDesignDir,
|
||||
maxTurns: 30,
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'design-review-fix',
|
||||
runId,
|
||||
});
|
||||
@@ -634,7 +635,7 @@ Review the site at ${serverUrl}. Use --quick mode. Skip any AskUserQuestion call
|
||||
console.warn('No design-audit.md generated');
|
||||
}
|
||||
console.log(`Design fix commits: ${designFixCommits.length}`);
|
||||
}, 420_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
* with its preamble.
|
||||
*/
|
||||
import { describe, expect } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
@@ -73,7 +74,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring-
|
||||
workingDirectory: dir,
|
||||
maxTurns: 25,
|
||||
allowedTools: ['Bash', 'Read', 'Write'],
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'diagram-triplet',
|
||||
runId,
|
||||
});
|
||||
@@ -98,7 +99,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring-
|
||||
} finally {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testConcurrentIfSelected('diagram-authoring-quality', async () => {
|
||||
const dir = setupDir('diagram-quality-');
|
||||
@@ -111,7 +112,7 @@ describeIfSelected('/diagram skill E2E', ['diagram-triplet', 'diagram-authoring-
|
||||
workingDirectory: dir,
|
||||
maxTurns: 25,
|
||||
allowedTools: ['Bash', 'Read', 'Write'],
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'diagram-authoring-quality',
|
||||
runId,
|
||||
});
|
||||
@@ -149,5 +150,5 @@ Respond with JSON: {"score": N, "reasoning": "..."}`,
|
||||
} finally {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
*/
|
||||
|
||||
import { expect, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
@@ -55,7 +56,7 @@ async function detectVia(workDir: string, testName: string): Promise<string> {
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 3,
|
||||
allowedTools: ['Bash'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
model: MODEL,
|
||||
@@ -91,7 +92,7 @@ describeIfSelected('first-run scaffold detection (E2E)', ['first-task-scaffold']
|
||||
fs.rmSync(nodeDir, { recursive: true, force: true });
|
||||
fs.rmSync(greenDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
afterAll(() => finalizeEvalCollector(evalCollector));
|
||||
|
||||
@@ -29,6 +29,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS } from './helpers/eval-budgets';
|
||||
import { execFileSync } from 'child_process';
|
||||
import { mkdtempSync, rmSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
@@ -156,7 +157,7 @@ ${body}`;
|
||||
expect(retrieved).not.toContain('page_not_found');
|
||||
expect(retrieved).not.toContain('Page not found');
|
||||
},
|
||||
120_000,
|
||||
JUDGE_MS,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
*/
|
||||
|
||||
import { expect, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
@@ -94,7 +95,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 3,
|
||||
allowedTools: ['Bash'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'hermetic-canary',
|
||||
runId,
|
||||
model: CANARY_MODEL,
|
||||
@@ -129,7 +130,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
|
||||
}
|
||||
fs.rmSync(workDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testIfSelected('hermetic-sentinel', async () => {
|
||||
if (!process.env.ANTHROPIC_API_KEY) {
|
||||
@@ -158,7 +159,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 3,
|
||||
allowedTools: ['Bash'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'hermetic-sentinel',
|
||||
runId,
|
||||
model: CANARY_MODEL,
|
||||
@@ -188,7 +189,7 @@ describeIfSelected('hermetic isolation canaries', ['hermetic-canary', 'hermetic-
|
||||
fs.rmSync(workDir, { recursive: true, force: true });
|
||||
fs.rmSync(poisonRoot, { recursive: true, force: true });
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
afterAll(() => finalizeEvalCollector(evalCollector));
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
// intentionally machine-specific.
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync, unlinkSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
@@ -856,7 +857,7 @@ describe('ios device deployment (explicit opt-in)', () => {
|
||||
keepalive?.stop();
|
||||
rmSync(workDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 600_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// Always-on instructions if not paired. Surfaces actionable steps even when
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
// gated (no compilation step for DebugBridgeCore/UI)
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
@@ -321,7 +322,7 @@ describeIfSwift('swift build invariants', () => {
|
||||
console.error('swift build stderr:', r.stderr?.toString().slice(0, 4000));
|
||||
}
|
||||
expect(r.status).toBe(0);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
test('XCTest suite for StateServer passes (validates real Swift impl)', () => {
|
||||
const r = spawnSync('swift', ['test', '--filter', 'DebugBridgeCoreTests'], {
|
||||
@@ -342,7 +343,7 @@ describeIfSwift('swift build invariants', () => {
|
||||
// Guard against an empty pass-by-no-tests (filter typo / target rename):
|
||||
// we expect at least one StateServer smoke test to actually execute.
|
||||
expect(combined).toContain('StateServerSmokeTests');
|
||||
}, 240_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// Codex-flagged: Release-build guard must be STRUCTURAL, not advisory.
|
||||
// The Package.swift's `.when(configuration: .debug)` setting causes Swift
|
||||
@@ -386,5 +387,5 @@ describeIfSwift('swift build invariants', () => {
|
||||
}
|
||||
}
|
||||
expect(foundForbidden).toBe(0);
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId, evalsEnabled,
|
||||
@@ -103,7 +104,7 @@ IMPORTANT:
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'learnings-show',
|
||||
runId,
|
||||
});
|
||||
@@ -134,5 +135,5 @@ IMPORTANT:
|
||||
} else {
|
||||
console.warn(`Only ${foundCount}/3 learnings found (N+1: ${mentionsNPlusOne}, cache: ${mentionsCache}, rubocop: ${mentionsRubocop})`);
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillObservation, planFileHasDecisionsSection } from './helpers/claude-pty-runner';
|
||||
|
||||
@@ -30,7 +31,7 @@ describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => {
|
||||
skillName: 'office-hours',
|
||||
inPlanMode: true,
|
||||
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (
|
||||
@@ -55,5 +56,5 @@ describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => {
|
||||
}
|
||||
}
|
||||
expect(['asked', 'plan_ready']).toContain(obs.outcome);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -36,6 +36,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { execFileSync, spawnSync } from 'child_process';
|
||||
import {
|
||||
chmodSync,
|
||||
@@ -122,6 +123,8 @@ describeIfSelected(
|
||||
'--respect-detection',
|
||||
],
|
||||
{
|
||||
// LIVE-REPO CWD: gen-skill-docs regenerates the in-repo
|
||||
// office-hours SKILL.md + section (snapshotted/restored in finally).
|
||||
cwd: ROOT,
|
||||
env: { ...process.env, GSTACK_HOME: tmpHome },
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
@@ -220,7 +223,7 @@ Generate the design doc per Phase 5. The feature-slug value to substitute into t
|
||||
This is a test of the brain-writeback path. Do NOT skip the gbrain save step under any circumstance — the runtime guard ("skip if gbrain not on PATH") does NOT apply here because gbrain IS available. Do NOT explore gbrain --help; follow the SAVE_RESULTS template's exact CLI shape. If you encounter any AskUserQuestion, auto-decide recommended.`,
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 12,
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'office-hours-brain-writeback',
|
||||
runId,
|
||||
model: 'claude-sonnet-4-6',
|
||||
@@ -313,7 +316,7 @@ This is a test of the brain-writeback path. Do NOT skip the gbrain save step und
|
||||
);
|
||||
}
|
||||
},
|
||||
420_000,
|
||||
CAPTURE_LONG_MS,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
* test turns out stable.
|
||||
*/
|
||||
import { expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
@@ -133,7 +134,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file with that ONE Phase 4 question, stop. Do not continue to Phase 4.5 or Phase 5.`,
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 12,
|
||||
timeout: 300_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'office-hours-phase4-fork',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -162,7 +163,7 @@ After writing the file with that ONE Phase 4 question, stop. Do not continue to
|
||||
result,
|
||||
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
||||
});
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled,
|
||||
@@ -71,7 +72,7 @@ Assume the founder has already answered Q1 (strongest evidence = "got on a waitl
|
||||
Write Q3 output — the forcing question you would ask this founder — to ${workDir}/q3.md. Write ONLY the question prose. No conversational wrapper, no meta-commentary, no Q1/Q2 recap.`,
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 8,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'office-hours-forcing-energy',
|
||||
runId,
|
||||
model: 'claude-sonnet-4-6',
|
||||
@@ -94,7 +95,7 @@ Write Q3 output — the forcing question you would ask this founder — to ${wor
|
||||
console.log('Forcing energy scores:', JSON.stringify(scores, null, 2));
|
||||
expect(scores.axis_a).toBeGreaterThanOrEqual(4); // stacking_preserved
|
||||
expect(scores.axis_b).toBeGreaterThanOrEqual(4); // domain_matched_consequence
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Office Hours builder-mode wildness ---
|
||||
@@ -143,7 +144,7 @@ The user has confirmed the basic idea is "TypeScript + D3 web tool, start with J
|
||||
Write your response — the three adjacent unlocks — to ${workDir}/unlocks.md. Write ONLY the response prose. No meta-commentary, no mode recap. Lead with the fun; let me edit it down later.`,
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 8,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'office-hours-builder-wildness',
|
||||
runId,
|
||||
model: 'claude-sonnet-4-6',
|
||||
@@ -166,7 +167,7 @@ Write your response — the three adjacent unlocks — to ${workDir}/unlocks.md.
|
||||
console.log('Builder wildness scores:', JSON.stringify(scores, null, 2));
|
||||
expect(scores.axis_a).toBeGreaterThanOrEqual(4); // unexpected_combinations
|
||||
expect(scores.axis_b).toBeGreaterThanOrEqual(4); // excitement_over_optimization
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// Finalize eval collector for this file
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import { EvalCollector } from './helpers/eval-store';
|
||||
import { extractSkillHead } from './helpers/skill-fixture';
|
||||
@@ -67,6 +68,8 @@ function mkEvalRoot(suffix: string, includeOverlay: boolean): string {
|
||||
const result = spawnSync(
|
||||
'bun',
|
||||
['run', 'scripts/gen-skill-docs.ts', '--model', includeOverlay ? 'opus-4-7' : 'claude'],
|
||||
// LIVE-REPO CWD: gen-skill-docs reads .tmpl sources and regenerates the
|
||||
// in-repo SKILL.md files (restored to default in afterAll below).
|
||||
{ cwd: ROOT, stdio: 'pipe', encoding: 'utf-8', timeout: 60_000 },
|
||||
);
|
||||
if (result.status !== 0) {
|
||||
@@ -169,6 +172,8 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
|
||||
// whichever model ran last. Reset to the default (claude) so the tree
|
||||
// matches what would be checked in.
|
||||
spawnSync('bun', ['run', 'scripts/gen-skill-docs.ts'], {
|
||||
// LIVE-REPO CWD: restores the in-repo SKILL.md files to the default
|
||||
// model render after mkEvalRoot's --model regens.
|
||||
cwd: ROOT,
|
||||
stdio: 'pipe',
|
||||
timeout: 60_000,
|
||||
@@ -200,7 +205,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
|
||||
workingDirectory: armA,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'fanout-arm-overlay-on',
|
||||
runId,
|
||||
model: OPUS_47,
|
||||
@@ -210,7 +215,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
|
||||
workingDirectory: armB,
|
||||
maxTurns: 5,
|
||||
allowedTools: ['Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'fanout-arm-overlay-off',
|
||||
runId,
|
||||
model: OPUS_47,
|
||||
@@ -258,7 +263,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
|
||||
fs.rmSync(armB, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
240_000,
|
||||
CAPTURE_MS,
|
||||
);
|
||||
|
||||
test(
|
||||
@@ -277,7 +282,7 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
|
||||
workingDirectory: root,
|
||||
maxTurns: 3,
|
||||
allowedTools: ['Skill', 'Read', 'Bash', 'Glob', 'Grep'],
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: `routing-${c.name}`,
|
||||
runId,
|
||||
model: OPUS_47,
|
||||
@@ -344,6 +349,6 @@ describeE2E('Opus 4.7 overlay behavior evals', () => {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
},
|
||||
360_000,
|
||||
CAPTURE_LONG_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -18,6 +18,8 @@
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import {
|
||||
runPlanSkillCounting,
|
||||
ceoStep0Boundary,
|
||||
@@ -62,8 +64,8 @@ const N_PAIRED = 2;
|
||||
const FLOOR_PAIRED = 2;
|
||||
const CEILING_PAIRED = 4;
|
||||
|
||||
const PLAN_CEO_5_FINDINGS = [
|
||||
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-ceo.md (use Edit/Write to that exact path).',
|
||||
const planCeo5Findings = (planPath: string) => [
|
||||
`Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
|
||||
'',
|
||||
'# Plan: Payment Processing Integration',
|
||||
'',
|
||||
@@ -88,8 +90,8 @@ const PLAN_CEO_5_FINDINGS = [
|
||||
'order in a loop.',
|
||||
].join('\n');
|
||||
|
||||
const PLAN_CEO_2_PAIRED_FINDINGS = [
|
||||
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-ceo-paired.md (use Edit/Write to that exact path).',
|
||||
const planCeo2PairedFindings = (planPath: string) => [
|
||||
`Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
|
||||
'',
|
||||
'# Plan: Payment Processing — Test Coverage',
|
||||
'',
|
||||
@@ -102,32 +104,31 @@ const PLAN_CEO_2_PAIRED_FINDINGS = [
|
||||
'the success path is correctness, the failure path is graceful degradation.',
|
||||
].join('\n');
|
||||
|
||||
const PLAN_CEO_PATH = '/tmp/gstack-test-plan-ceo.md';
|
||||
const PLAN_CEO_PAIRED_PATH = '/tmp/gstack-test-plan-ceo-paired.md';
|
||||
|
||||
describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', () => {
|
||||
test(
|
||||
`5-finding plan emits ${FLOOR_DISTINCT}-${CEILING_DISTINCT} review-phase AskUserQuestions`,
|
||||
async () => {
|
||||
try {
|
||||
fs.rmSync(PLAN_CEO_PATH, { force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-ceo-review',
|
||||
slashCommand: '/plan-ceo-review',
|
||||
followUpPrompt: PLAN_CEO_5_FINDINGS,
|
||||
isLastStep0AUQ: ceoStep0Boundary,
|
||||
reviewCountCeiling: CEILING_DISTINCT + 1, // hard cap above assertion ceiling
|
||||
firstAUQPick: pickSkipInterview, // bypass scope-selection, route to review
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000, // 25 min
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
// Per-run artifact dir: a hardcoded shared /tmp path collides under
|
||||
// --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally-
|
||||
// rmSync deletes this run's artifact → spurious D19 failure).
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-'));
|
||||
const planPath = path.join(tmpDir, 'gstack-test-plan-ceo.md');
|
||||
|
||||
try {
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-ceo-review',
|
||||
slashCommand: '/plan-ceo-review',
|
||||
followUpPrompt: planCeo5Findings(planPath),
|
||||
isLastStep0AUQ: ceoStep0Boundary,
|
||||
reviewCountCeiling: CEILING_DISTINCT + 1, // hard cap above assertion ceiling
|
||||
firstAUQPick: pickSkipInterview, // bypass scope-selection, route to review
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000, // 25 min
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
|
||||
throw new Error(
|
||||
`plan-ceo-review finding-count FAILED: outcome=${obs.outcome}\n` +
|
||||
@@ -166,19 +167,19 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
|
||||
}
|
||||
|
||||
// D19: review report at bottom of plan file.
|
||||
if (!fs.existsSync(PLAN_CEO_PATH)) {
|
||||
if (!fs.existsSync(planPath)) {
|
||||
throw new Error(
|
||||
`D19 FAIL: agent did not produce expected plan file at ${PLAN_CEO_PATH}.\n` +
|
||||
`D19 FAIL: agent did not produce expected plan file at ${planPath}.\n` +
|
||||
`Either the agent ignored the path instruction in the follow-up prompt, or\n` +
|
||||
`the helper exited before the agent wrote the file. ` +
|
||||
`outcome=${obs.outcome} review=${obs.reviewCount}`,
|
||||
);
|
||||
}
|
||||
const planContent = fs.readFileSync(PLAN_CEO_PATH, 'utf-8');
|
||||
const planContent = fs.readFileSync(planPath, 'utf-8');
|
||||
const verdict = assertReviewReportAtBottom(planContent);
|
||||
if (!verdict.ok) {
|
||||
throw new Error(
|
||||
`D19 FAIL: plan file at ${PLAN_CEO_PATH} ${verdict.reason}\n` +
|
||||
`D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` +
|
||||
(verdict.trailingHeadings
|
||||
? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n`
|
||||
: '') +
|
||||
@@ -187,36 +188,36 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
fs.rmSync(PLAN_CEO_PATH, { force: true });
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
}
|
||||
},
|
||||
1_700_000,
|
||||
1_500_000 /* physical ceiling: the 25-min CI job + 1800s shard wall cap what can actually execute */,
|
||||
);
|
||||
|
||||
test(
|
||||
`paired-finding positive control: ${N_PAIRED} related findings produce ${FLOOR_PAIRED}-${CEILING_PAIRED} AskUserQuestions`,
|
||||
async () => {
|
||||
try {
|
||||
fs.rmSync(PLAN_CEO_PAIRED_PATH, { force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-ceo-review',
|
||||
slashCommand: '/plan-ceo-review',
|
||||
followUpPrompt: PLAN_CEO_2_PAIRED_FINDINGS,
|
||||
isLastStep0AUQ: ceoStep0Boundary,
|
||||
reviewCountCeiling: CEILING_PAIRED + 1,
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
// Per-run artifact dir — see the distinct-findings test above.
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-paired-'));
|
||||
const planPath = path.join(tmpDir, 'gstack-test-plan-ceo-paired.md');
|
||||
|
||||
try {
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-ceo-review',
|
||||
slashCommand: '/plan-ceo-review',
|
||||
followUpPrompt: planCeo2PairedFindings(planPath),
|
||||
isLastStep0AUQ: ceoStep0Boundary,
|
||||
reviewCountCeiling: CEILING_PAIRED + 1,
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
|
||||
throw new Error(
|
||||
`paired-finding control FAILED: outcome=${obs.outcome}\n` +
|
||||
@@ -242,12 +243,12 @@ describeE2E('/plan-ceo-review per-finding AskUserQuestion count (periodic)', ()
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
fs.rmSync(PLAN_CEO_PAIRED_PATH, { force: true });
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
}
|
||||
},
|
||||
1_700_000,
|
||||
1_500_000 /* physical ceiling: the 25-min CI job + 1800s shard wall cap what can actually execute */,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
*/
|
||||
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
|
||||
import { FORCING_FLOOR_CEO } from './fixtures/forcing-finding-seeds';
|
||||
@@ -19,8 +20,10 @@ describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => {
|
||||
skillName: 'plan-ceo-review',
|
||||
slashCommand: '/plan-ceo-review',
|
||||
followUpPrompt: FORCING_FLOOR_CEO,
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 600_000,
|
||||
timeoutMs: CAPTURE_LONG_MS,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
@@ -32,6 +35,6 @@ describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => {
|
||||
);
|
||||
}
|
||||
},
|
||||
660_000,
|
||||
PTY_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
*/
|
||||
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
launchClaudePty,
|
||||
@@ -151,7 +152,7 @@ describeE2E('/plan-ceo-review mode routing (gate)', () => {
|
||||
async () => {
|
||||
const session = await launchClaudePty({
|
||||
permissionMode: 'plan',
|
||||
timeoutMs: 540_000,
|
||||
timeoutMs: CAPTURE_LONG_MS,
|
||||
seedSkills: true,
|
||||
});
|
||||
try {
|
||||
@@ -207,7 +208,7 @@ describeE2E('/plan-ceo-review mode routing (gate)', () => {
|
||||
await session.close();
|
||||
}
|
||||
},
|
||||
600_000,
|
||||
CAPTURE_LONG_MS,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -34,6 +34,7 @@
|
||||
*/
|
||||
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
runPlanSkillObservation,
|
||||
@@ -77,5 +78,5 @@ describeE2E('plan-ceo-review plan-mode smoke (gate)', () => {
|
||||
);
|
||||
}
|
||||
assertReportAtBottomIfPlanWritten(obs);
|
||||
}, 480_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
setupSkillDir,
|
||||
@@ -87,6 +88,6 @@ describeE2E('/plan-ceo-review section-loading E2E (periodic, SDK capture)', () =
|
||||
// Guard against an empty pass: the report must have real content.
|
||||
expect(output.trim().length).toBeGreaterThan(200);
|
||||
},
|
||||
360_000,
|
||||
CAPTURE_LONG_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -35,6 +35,8 @@
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import {
|
||||
runPlanSkillCounting,
|
||||
ceoStep0Boundary,
|
||||
@@ -46,30 +48,38 @@ const describeE2E = describeE2ETier('periodic');
|
||||
const N = 5;
|
||||
const FLOOR = N - 1; // 4 — must fire at least one AUQ per non-dropped option
|
||||
|
||||
const PLAN_PATH = '/tmp/gstack-test-plan-ceo-split-overflow.md';
|
||||
/** Plan-file target baked into the FORCING_SPLIT_OVERFLOW_CEO fixture prompt.
|
||||
* Rewritten per-run to a mkdtemp path so concurrent runs (--retry,
|
||||
* EVALS_JOBS>1, sibling worktrees) never share one /tmp artifact. */
|
||||
const FIXTURE_PLAN_PATH = '/tmp/gstack-test-plan-ceo-split-overflow.md';
|
||||
|
||||
describeE2E('/plan-ceo-review split-overflow regression (periodic)', () => {
|
||||
test(
|
||||
`5-option scope decision emits >= ${FLOOR} review-phase AskUserQuestions (no dropping)`,
|
||||
async () => {
|
||||
try {
|
||||
fs.rmSync(PLAN_PATH, { force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-ceo-split-overflow-'));
|
||||
const planPath = path.join(tmpDir, 'gstack-test-plan-ceo-split-overflow.md');
|
||||
const followUpPrompt = FORCING_SPLIT_OVERFLOW_CEO.replaceAll(FIXTURE_PLAN_PATH, planPath);
|
||||
if (!followUpPrompt.includes(planPath)) {
|
||||
throw new Error(
|
||||
`fixture drift: FORCING_SPLIT_OVERFLOW_CEO no longer contains ${FIXTURE_PLAN_PATH} — update FIXTURE_PLAN_PATH`,
|
||||
);
|
||||
}
|
||||
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-ceo-review',
|
||||
slashCommand: '/plan-ceo-review',
|
||||
followUpPrompt: FORCING_SPLIT_OVERFLOW_CEO,
|
||||
isLastStep0AUQ: ceoStep0Boundary,
|
||||
reviewCountCeiling: N + 3, // hard cap above floor + tolerance
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000, // 25 min
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
try {
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-ceo-review',
|
||||
slashCommand: '/plan-ceo-review',
|
||||
followUpPrompt,
|
||||
isLastStep0AUQ: ceoStep0Boundary,
|
||||
reviewCountCeiling: N + 3, // hard cap above floor + tolerance
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000, // 25 min
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
|
||||
throw new Error(
|
||||
`split-overflow test FAILED: outcome=${obs.outcome}\n` +
|
||||
@@ -97,12 +107,12 @@ describeE2E('/plan-ceo-review split-overflow regression (periodic)', () => {
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
fs.rmSync(PLAN_PATH, { force: true });
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
}
|
||||
},
|
||||
1_700_000,
|
||||
1_500_000 /* physical ceiling: the 25-min CI job + 1800s shard wall cap what can actually execute */,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -11,6 +11,8 @@
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import {
|
||||
runPlanSkillCounting,
|
||||
designStep0Boundary,
|
||||
@@ -23,8 +25,8 @@ const N = 5;
|
||||
const FLOOR = N - 1;
|
||||
const CEILING = N + 2;
|
||||
|
||||
const PLAN_DESIGN_5_FINDINGS = [
|
||||
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-design.md (use Edit/Write to that exact path).',
|
||||
const planDesign5Findings = (planPath: string) => [
|
||||
`Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
|
||||
'',
|
||||
'# Plan: Settings Page UI redesign',
|
||||
'',
|
||||
@@ -50,30 +52,30 @@ const PLAN_DESIGN_5_FINDINGS = [
|
||||
'see a frozen page; we should add a spinner or skeleton state.',
|
||||
].join('\n');
|
||||
|
||||
const PLAN_DESIGN_PATH = '/tmp/gstack-test-plan-design.md';
|
||||
|
||||
describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)', () => {
|
||||
test(
|
||||
`5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`,
|
||||
async () => {
|
||||
try {
|
||||
fs.rmSync(PLAN_DESIGN_PATH, { force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-design-review',
|
||||
slashCommand: '/plan-design-review',
|
||||
followUpPrompt: PLAN_DESIGN_5_FINDINGS,
|
||||
isLastStep0AUQ: designStep0Boundary,
|
||||
reviewCountCeiling: CEILING + 1,
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
// Per-run artifact dir: a hardcoded shared /tmp path collides under
|
||||
// --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally-
|
||||
// rmSync deletes this run's artifact → spurious D19 failure).
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-design-'));
|
||||
const planPath = path.join(tmpDir, 'gstack-test-plan-design.md');
|
||||
|
||||
try {
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-design-review',
|
||||
slashCommand: '/plan-design-review',
|
||||
followUpPrompt: planDesign5Findings(planPath),
|
||||
isLastStep0AUQ: designStep0Boundary,
|
||||
reviewCountCeiling: CEILING + 1,
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
|
||||
throw new Error(
|
||||
`plan-design-review finding-count FAILED: outcome=${obs.outcome}\n` +
|
||||
@@ -105,17 +107,17 @@ describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)',
|
||||
);
|
||||
}
|
||||
|
||||
if (!fs.existsSync(PLAN_DESIGN_PATH)) {
|
||||
if (!fs.existsSync(planPath)) {
|
||||
throw new Error(
|
||||
`D19 FAIL: agent did not produce expected plan file at ${PLAN_DESIGN_PATH}. ` +
|
||||
`D19 FAIL: agent did not produce expected plan file at ${planPath}. ` +
|
||||
`outcome=${obs.outcome} review=${obs.reviewCount}`,
|
||||
);
|
||||
}
|
||||
const planContent = fs.readFileSync(PLAN_DESIGN_PATH, 'utf-8');
|
||||
const planContent = fs.readFileSync(planPath, 'utf-8');
|
||||
const verdict = assertReviewReportAtBottom(planContent);
|
||||
if (!verdict.ok) {
|
||||
throw new Error(
|
||||
`D19 FAIL: plan file at ${PLAN_DESIGN_PATH} ${verdict.reason}\n` +
|
||||
`D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` +
|
||||
(verdict.trailingHeadings
|
||||
? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n`
|
||||
: '') +
|
||||
@@ -124,12 +126,12 @@ describeE2E('/plan-design-review per-finding AskUserQuestion count (periodic)',
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
fs.rmSync(PLAN_DESIGN_PATH, { force: true });
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
}
|
||||
},
|
||||
1_700_000,
|
||||
1_500_000 /* physical ceiling: the 25-min CI job + 1800s shard wall cap what can actually execute */,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
*/
|
||||
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
|
||||
import { FORCING_FLOOR_DESIGN } from './fixtures/forcing-finding-seeds';
|
||||
@@ -19,8 +20,10 @@ describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => {
|
||||
skillName: 'plan-design-review',
|
||||
slashCommand: '/plan-design-review',
|
||||
followUpPrompt: FORCING_FLOOR_DESIGN,
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 600_000,
|
||||
timeoutMs: CAPTURE_LONG_MS,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
@@ -32,6 +35,6 @@ describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => {
|
||||
);
|
||||
}
|
||||
},
|
||||
660_000,
|
||||
PTY_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
runPlanSkillObservation,
|
||||
@@ -46,7 +47,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
|
||||
const obs = await runPlanSkillObservation({
|
||||
skillName: 'plan-design-review',
|
||||
inPlanMode: true,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
|
||||
@@ -59,7 +60,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
|
||||
}
|
||||
expect(['asked', 'plan_ready']).toContain(obs.outcome);
|
||||
assertReportAtBottomIfPlanWritten(obs);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
// Plan-mode scope-gate bypass: with a seeded UI-heavy plan in plan mode,
|
||||
// the gate must NOT render its "What should I review?" menu — it
|
||||
@@ -71,7 +72,7 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
|
||||
skillName: 'plan-design-review',
|
||||
inPlanMode: true,
|
||||
initialPlanContent: SEED_PLAN_UI_HEAVY,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (
|
||||
@@ -95,5 +96,5 @@ describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
|
||||
// though the seed arrives as a pasted user message).
|
||||
expect(obs.scopeGateQuestionObserved ?? false).toBe(false);
|
||||
expect(obs.scopeGateAutoSelectObserved ?? false).toBe(true);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
*/
|
||||
|
||||
import { test } from 'bun:test';
|
||||
import { PTY_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as path from 'path';
|
||||
import {
|
||||
@@ -43,8 +44,10 @@ describeE2E('/plan-design-review with UI scope (gate)', () => {
|
||||
|
||||
const session = await launchClaudePty({
|
||||
permissionMode: 'plan',
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — skill registry,
|
||||
// hermetic pre-trusted dir, and the repo-relative fixture path above.
|
||||
cwd: ROOT,
|
||||
timeoutMs: 720_000,
|
||||
timeoutMs: PTY_MS,
|
||||
seedSkills: true,
|
||||
});
|
||||
|
||||
@@ -150,6 +153,6 @@ describeE2E('/plan-design-review with UI scope (gate)', () => {
|
||||
);
|
||||
}
|
||||
},
|
||||
780_000,
|
||||
PTY_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -11,6 +11,8 @@
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import {
|
||||
runPlanSkillCounting,
|
||||
devexStep0Boundary,
|
||||
@@ -23,8 +25,8 @@ const N = 5;
|
||||
const FLOOR = N - 1;
|
||||
const CEILING = N + 2;
|
||||
|
||||
const PLAN_DEVEX_5_FINDINGS = [
|
||||
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-devex.md (use Edit/Write to that exact path).',
|
||||
const planDevex5Findings = (planPath: string) => [
|
||||
`Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
|
||||
'',
|
||||
'# Plan: Public SDK Beta Launch',
|
||||
'',
|
||||
@@ -50,30 +52,30 @@ const PLAN_DEVEX_5_FINDINGS = [
|
||||
'of solved problems.',
|
||||
].join('\n');
|
||||
|
||||
const PLAN_DEVEX_PATH = '/tmp/gstack-test-plan-devex.md';
|
||||
|
||||
describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', () => {
|
||||
test(
|
||||
`5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`,
|
||||
async () => {
|
||||
try {
|
||||
fs.rmSync(PLAN_DEVEX_PATH, { force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-devex-review',
|
||||
slashCommand: '/plan-devex-review',
|
||||
followUpPrompt: PLAN_DEVEX_5_FINDINGS,
|
||||
isLastStep0AUQ: devexStep0Boundary,
|
||||
reviewCountCeiling: CEILING + 1,
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
// Per-run artifact dir: a hardcoded shared /tmp path collides under
|
||||
// --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally-
|
||||
// rmSync deletes this run's artifact → spurious D19 failure).
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-devex-'));
|
||||
const planPath = path.join(tmpDir, 'gstack-test-plan-devex.md');
|
||||
|
||||
try {
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-devex-review',
|
||||
slashCommand: '/plan-devex-review',
|
||||
followUpPrompt: planDevex5Findings(planPath),
|
||||
isLastStep0AUQ: devexStep0Boundary,
|
||||
reviewCountCeiling: CEILING + 1,
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
|
||||
throw new Error(
|
||||
`plan-devex-review finding-count FAILED: outcome=${obs.outcome}\n` +
|
||||
@@ -105,17 +107,17 @@ describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', (
|
||||
);
|
||||
}
|
||||
|
||||
if (!fs.existsSync(PLAN_DEVEX_PATH)) {
|
||||
if (!fs.existsSync(planPath)) {
|
||||
throw new Error(
|
||||
`D19 FAIL: agent did not produce expected plan file at ${PLAN_DEVEX_PATH}. ` +
|
||||
`D19 FAIL: agent did not produce expected plan file at ${planPath}. ` +
|
||||
`outcome=${obs.outcome} review=${obs.reviewCount}`,
|
||||
);
|
||||
}
|
||||
const planContent = fs.readFileSync(PLAN_DEVEX_PATH, 'utf-8');
|
||||
const planContent = fs.readFileSync(planPath, 'utf-8');
|
||||
const verdict = assertReviewReportAtBottom(planContent);
|
||||
if (!verdict.ok) {
|
||||
throw new Error(
|
||||
`D19 FAIL: plan file at ${PLAN_DEVEX_PATH} ${verdict.reason}\n` +
|
||||
`D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` +
|
||||
(verdict.trailingHeadings
|
||||
? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n`
|
||||
: '') +
|
||||
@@ -124,12 +126,12 @@ describeE2E('/plan-devex-review per-finding AskUserQuestion count (periodic)', (
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
fs.rmSync(PLAN_DEVEX_PATH, { force: true });
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
}
|
||||
},
|
||||
1_700_000,
|
||||
1_500_000 /* physical ceiling: the 25-min CI job + 1800s shard wall cap what can actually execute */,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
*/
|
||||
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
|
||||
import { FORCING_FLOOR_DEVEX } from './fixtures/forcing-finding-seeds';
|
||||
@@ -19,8 +20,10 @@ describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => {
|
||||
skillName: 'plan-devex-review',
|
||||
slashCommand: '/plan-devex-review',
|
||||
followUpPrompt: FORCING_FLOOR_DEVEX,
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 600_000,
|
||||
timeoutMs: CAPTURE_LONG_MS,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
@@ -32,6 +35,6 @@ describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => {
|
||||
);
|
||||
}
|
||||
},
|
||||
660_000,
|
||||
PTY_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
runPlanSkillObservation,
|
||||
@@ -20,7 +21,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
|
||||
const obs = await runPlanSkillObservation({
|
||||
skillName: 'plan-devex-review',
|
||||
inPlanMode: true,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
|
||||
@@ -33,7 +34,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
|
||||
}
|
||||
expect(['asked', 'plan_ready']).toContain(obs.outcome);
|
||||
assertReportAtBottomIfPlanWritten(obs);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
// v1.21+ regression: see skill-e2e-plan-ceo-plan-mode.test.ts for the
|
||||
// contract. Pass envelope is ['asked', 'plan_ready']; failure signals
|
||||
@@ -44,7 +45,7 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
|
||||
skillName: 'plan-devex-review',
|
||||
inPlanMode: true,
|
||||
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (
|
||||
@@ -70,5 +71,5 @@ describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
|
||||
}
|
||||
expect(['asked', 'plan_ready']).toContain(obs.outcome);
|
||||
assertReportAtBottomIfPlanWritten(obs);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -11,6 +11,8 @@
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import {
|
||||
runPlanSkillCounting,
|
||||
engStep0Boundary,
|
||||
@@ -23,8 +25,8 @@ const N = 5;
|
||||
const FLOOR = N - 1; // 4
|
||||
const CEILING = N + 2; // 7
|
||||
|
||||
const PLAN_ENG_5_FINDINGS = [
|
||||
'Please review this plan thoroughly. As you go, write your plan-mode plan to /tmp/gstack-test-plan-eng.md (use Edit/Write to that exact path).',
|
||||
const planEng5Findings = (planPath: string) => [
|
||||
`Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
|
||||
'',
|
||||
'# Plan: Multi-tenant Auth Refactor',
|
||||
'',
|
||||
@@ -49,30 +51,30 @@ const PLAN_ENG_5_FINDINGS = [
|
||||
'SessionMint, AuthCache, RequestPolicy). Worth flagging the complexity check.',
|
||||
].join('\n');
|
||||
|
||||
const PLAN_ENG_PATH = '/tmp/gstack-test-plan-eng.md';
|
||||
|
||||
describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', () => {
|
||||
test(
|
||||
`5-finding plan emits ${FLOOR}-${CEILING} review-phase AskUserQuestions`,
|
||||
async () => {
|
||||
try {
|
||||
fs.rmSync(PLAN_ENG_PATH, { force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-eng-review',
|
||||
slashCommand: '/plan-eng-review',
|
||||
followUpPrompt: PLAN_ENG_5_FINDINGS,
|
||||
isLastStep0AUQ: engStep0Boundary,
|
||||
reviewCountCeiling: CEILING + 1,
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
// Per-run artifact dir: a hardcoded shared /tmp path collides under
|
||||
// --retry, EVALS_JOBS>1, or concurrent worktrees (a sibling's finally-
|
||||
// rmSync deletes this run's artifact → spurious D19 failure).
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-eng-'));
|
||||
const planPath = path.join(tmpDir, 'gstack-test-plan-eng.md');
|
||||
|
||||
try {
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-eng-review',
|
||||
slashCommand: '/plan-eng-review',
|
||||
followUpPrompt: planEng5Findings(planPath),
|
||||
isLastStep0AUQ: engStep0Boundary,
|
||||
reviewCountCeiling: CEILING + 1,
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
|
||||
throw new Error(
|
||||
`plan-eng-review finding-count FAILED: outcome=${obs.outcome}\n` +
|
||||
@@ -104,17 +106,17 @@ describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', ()
|
||||
);
|
||||
}
|
||||
|
||||
if (!fs.existsSync(PLAN_ENG_PATH)) {
|
||||
if (!fs.existsSync(planPath)) {
|
||||
throw new Error(
|
||||
`D19 FAIL: agent did not produce expected plan file at ${PLAN_ENG_PATH}. ` +
|
||||
`D19 FAIL: agent did not produce expected plan file at ${planPath}. ` +
|
||||
`outcome=${obs.outcome} review=${obs.reviewCount}`,
|
||||
);
|
||||
}
|
||||
const planContent = fs.readFileSync(PLAN_ENG_PATH, 'utf-8');
|
||||
const planContent = fs.readFileSync(planPath, 'utf-8');
|
||||
const verdict = assertReviewReportAtBottom(planContent);
|
||||
if (!verdict.ok) {
|
||||
throw new Error(
|
||||
`D19 FAIL: plan file at ${PLAN_ENG_PATH} ${verdict.reason}\n` +
|
||||
`D19 FAIL: plan file at ${planPath} ${verdict.reason}\n` +
|
||||
(verdict.trailingHeadings
|
||||
? `Trailing headings: ${verdict.trailingHeadings.join(' | ')}\n`
|
||||
: '') +
|
||||
@@ -123,12 +125,12 @@ describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', ()
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
fs.rmSync(PLAN_ENG_PATH, { force: true });
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
}
|
||||
},
|
||||
1_700_000,
|
||||
1_500_000 /* physical ceiling: the 25-min CI job + 1800s shard wall cap what can actually execute */,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
*/
|
||||
|
||||
import { test } from 'bun:test';
|
||||
import { CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
|
||||
import { FORCING_FLOOR_ENG } from './fixtures/forcing-finding-seeds';
|
||||
@@ -30,8 +31,10 @@ describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => {
|
||||
skillName: 'plan-eng-review',
|
||||
slashCommand: '/plan-eng-review',
|
||||
followUpPrompt: FORCING_FLOOR_ENG,
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 600_000,
|
||||
timeoutMs: CAPTURE_LONG_MS,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
@@ -47,6 +50,6 @@ describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => {
|
||||
);
|
||||
}
|
||||
},
|
||||
660_000,
|
||||
PTY_MS,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -27,6 +27,8 @@
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import {
|
||||
runPlanSkillCounting,
|
||||
engStep0Boundary,
|
||||
@@ -38,30 +40,38 @@ const describeE2E = describeE2ETier('periodic');
|
||||
const N = 4;
|
||||
const FLOOR = N - 1; // 3 — agent must fire at least one AUQ per non-batched finding
|
||||
|
||||
const PLAN_PATH = '/tmp/gstack-test-plan-eng-batching.md';
|
||||
/** Plan-file target baked into the FORCING_BATCHING_ENG fixture prompt.
|
||||
* Rewritten per-run to a mkdtemp path so concurrent runs (--retry,
|
||||
* EVALS_JOBS>1, sibling worktrees) never share one /tmp artifact. */
|
||||
const FIXTURE_PLAN_PATH = '/tmp/gstack-test-plan-eng-batching.md';
|
||||
|
||||
describeE2E('/plan-eng-review multi-finding batching regression (periodic)', () => {
|
||||
test(
|
||||
`4-finding plan emits >= ${FLOOR} review-phase AskUserQuestions (no batching)`,
|
||||
async () => {
|
||||
try {
|
||||
fs.rmSync(PLAN_PATH, { force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-eng-batching-'));
|
||||
const planPath = path.join(tmpDir, 'gstack-test-plan-eng-batching.md');
|
||||
const followUpPrompt = FORCING_BATCHING_ENG.replaceAll(FIXTURE_PLAN_PATH, planPath);
|
||||
if (!followUpPrompt.includes(planPath)) {
|
||||
throw new Error(
|
||||
`fixture drift: FORCING_BATCHING_ENG no longer contains ${FIXTURE_PLAN_PATH} — update FIXTURE_PLAN_PATH`,
|
||||
);
|
||||
}
|
||||
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-eng-review',
|
||||
slashCommand: '/plan-eng-review',
|
||||
followUpPrompt: FORCING_BATCHING_ENG,
|
||||
isLastStep0AUQ: engStep0Boundary,
|
||||
reviewCountCeiling: N + 3, // hard cap above floor + tolerance
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000, // 25 min
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
try {
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-eng-review',
|
||||
slashCommand: '/plan-eng-review',
|
||||
followUpPrompt,
|
||||
isLastStep0AUQ: engStep0Boundary,
|
||||
reviewCountCeiling: N + 3, // hard cap above floor + tolerance
|
||||
// LIVE-REPO CWD: PTY session needs the repo cwd — gstack skill
|
||||
// registry + hermetic pre-trusted dir (hermetic-env trustedDirs).
|
||||
cwd: process.cwd(),
|
||||
timeoutMs: 1_500_000, // 25 min
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
|
||||
if (!['plan_ready', 'completion_summary', 'ceiling_reached'].includes(obs.outcome)) {
|
||||
throw new Error(
|
||||
`multi-finding batching test FAILED: outcome=${obs.outcome}\n` +
|
||||
@@ -85,12 +95,12 @@ describeE2E('/plan-eng-review multi-finding batching regression (periodic)', ()
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
fs.rmSync(PLAN_PATH, { force: true });
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
}
|
||||
},
|
||||
1_700_000,
|
||||
1_500_000 /* physical ceiling: the 25-min CI job + 1800s shard wall cap what can actually execute */,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
runPlanSkillObservation,
|
||||
@@ -50,7 +51,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
|
||||
const obs = await runPlanSkillObservation({
|
||||
skillName: 'plan-eng-review',
|
||||
inPlanMode: true,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
|
||||
@@ -63,7 +64,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
|
||||
}
|
||||
expect(['asked', 'plan_ready']).toContain(obs.outcome);
|
||||
assertReportAtBottomIfPlanWritten(obs);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
// D3-B / D4-B: when a plan with guaranteed-finding-triggering complexity
|
||||
// is seeded, the skill MUST fire AskUserQuestion (or fall back to a
|
||||
@@ -79,7 +80,7 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
|
||||
// must use mcp__*__AskUserQuestion (outcome='asked') or fall back to
|
||||
// writing Decisions ('plan_ready').
|
||||
extraArgs: ['--disallowedTools', 'AskUserQuestion'],
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (
|
||||
@@ -118,5 +119,5 @@ describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
|
||||
// question.
|
||||
expect(obs.scopeGateQuestionObserved ?? false).toBe(false);
|
||||
expect(obs.scopeGateAutoSelectObserved ?? false).toBe(true);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
* accordingly.
|
||||
*/
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
@@ -134,7 +135,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file, stop. Do not continue the review.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 10,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-ceo-review-format-mode',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -160,7 +161,7 @@ After writing the file, stop. Do not continue the review.`,
|
||||
result,
|
||||
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
||||
});
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Case 2: plan-ceo-review approach menu (coverage-differentiated) ---
|
||||
@@ -191,7 +192,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file, stop. Do not continue the review.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 10,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-ceo-review-format-approach',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -216,7 +217,7 @@ After writing the file, stop. Do not continue the review.`,
|
||||
result,
|
||||
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
||||
});
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Case 3: plan-eng-review coverage-differentiated per-issue AskUserQuestion ---
|
||||
@@ -250,7 +251,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file with that ONE question, stop. Do not continue the review.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 10,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-eng-review-format-coverage',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -275,7 +276,7 @@ After writing the file with that ONE question, stop. Do not continue the review.
|
||||
result,
|
||||
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
||||
});
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Case 4: plan-eng-review kind-differentiated per-issue AskUserQuestion ---
|
||||
@@ -306,7 +307,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file with that ONE question, stop. Do not continue the review.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 10,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-eng-review-format-kind',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -332,7 +333,7 @@ After writing the file with that ONE question, stop. Do not continue the review.
|
||||
result,
|
||||
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
||||
});
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
*/
|
||||
|
||||
import { test, expect } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
|
||||
|
||||
@@ -62,7 +63,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
|
||||
const obs = await runPlanSkillObservation({
|
||||
skillName,
|
||||
inPlanMode: false,
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
// eng/design: force the prose-fallback path. The unconditional
|
||||
// gate-must-ask assert below pins the render shape the detector
|
||||
// anchors on, and only the --disallowedTools prose fallback makes
|
||||
@@ -115,7 +116,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
|
||||
);
|
||||
}
|
||||
}
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
}
|
||||
|
||||
// Named-target exception (outside plan mode): a pasted draft IS an
|
||||
@@ -130,7 +131,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
|
||||
inPlanMode: false,
|
||||
initialPlanContent: NAMED_TARGET_SEED,
|
||||
trackTokens: [SEED_TOKEN],
|
||||
timeoutMs: 300_000,
|
||||
timeoutMs: CAPTURE_MS,
|
||||
});
|
||||
|
||||
if (
|
||||
@@ -159,5 +160,5 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
|
||||
// unreachable outside plan mode (extractPlanFilePath only matches
|
||||
// plan-mode save renders).
|
||||
expect(obs.tokensObserved?.[SEED_TOKEN] ?? false).toBe(true);
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
* cases will land as follow-up PRs per skill.
|
||||
*/
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
@@ -161,7 +162,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file, stop.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 10,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-review-prosons-format',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -191,7 +192,7 @@ After writing the file, stop.`,
|
||||
|
||||
// (recommended) label on one option
|
||||
expect(captured).toMatch(RECOMMENDED_LABEL_RE);
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Case 2: Hard-stop escape NEGATIVE (CT2) ---
|
||||
@@ -220,7 +221,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file, stop.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 10,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-review-prosons-hardstop-neg',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -241,7 +242,7 @@ After writing the file, stop.`,
|
||||
// Must have real pros and cons (≥2 ✅ + ≥1 ❌ per option)
|
||||
expect(countChars(captured, '✅')).toBeGreaterThanOrEqual(4);
|
||||
expect(countChars(captured, '❌')).toBeGreaterThanOrEqual(2);
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Case 3: Neutral-posture NEGATIVE (CT2) ---
|
||||
@@ -270,7 +271,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file, stop.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 10,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-review-prosons-neutral-neg',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -292,7 +293,7 @@ After writing the file, stop.`,
|
||||
expect(captured).toMatch(RECOMMENDED_LABEL_RE);
|
||||
// Recommendation line must contain "because" (concrete reason, not "no preference")
|
||||
expect(captured).toMatch(/[Rr]ecommendation:.*because/);
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Case 4: Hard-stop POSITIVE (escape allowed when legitimately one-sided) ---
|
||||
@@ -321,7 +322,7 @@ ${captureInstruction(outFile)}
|
||||
After writing the file, stop.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 10,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'plan-ceo-review-prosons-cadence',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -344,7 +345,7 @@ After writing the file, stop.`,
|
||||
const hasEscape = HARD_STOP_ESCAPE_RE.test(captured);
|
||||
const hasProsAndCons = countChars(captured, '✅') >= 1 && countChars(captured, '❌') >= 1;
|
||||
expect(hasEscape || hasProsAndCons).toBe(true);
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { beforeAll, afterAll, expect } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
@@ -151,7 +152,7 @@ IMPORTANT:
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'plan-tune-inspect',
|
||||
runId,
|
||||
});
|
||||
@@ -184,5 +185,5 @@ IMPORTANT:
|
||||
if (!noticedOverride) {
|
||||
console.warn('Agent did not surface override/skip behavior from the log');
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
+21
-20
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled,
|
||||
@@ -103,7 +104,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and
|
||||
const review = fs.readFileSync(reviewPath, 'utf-8');
|
||||
expect(review.length).toBeGreaterThan(200);
|
||||
}
|
||||
}, 660_000);
|
||||
}, PTY_MS);
|
||||
});
|
||||
|
||||
// --- Plan CEO Review (SELECTIVE EXPANSION) E2E ---
|
||||
@@ -171,7 +172,7 @@ Write your complete review directly to ${planDir}/review-output-selective.md
|
||||
Focus on reviewing the plan content: architecture, error handling, security, and performance.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 15,
|
||||
timeout: 540_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'plan-ceo-review-selective',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -188,7 +189,7 @@ Focus on reviewing the plan content: architecture, error handling, security, and
|
||||
const review = fs.readFileSync(reviewPath, 'utf-8');
|
||||
expect(review.length).toBeGreaterThan(200);
|
||||
}
|
||||
}, 660_000);
|
||||
}, PTY_MS);
|
||||
});
|
||||
|
||||
// --- Plan CEO Review SCOPE EXPANSION energy (V1.1 mode-posture regression gate) ---
|
||||
@@ -239,7 +240,7 @@ Choose SCOPE EXPANSION mode. Skip any AskUserQuestion calls — this is non-inte
|
||||
Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal text — no conversational wrapper, no review summary, no mode analysis. Each proposal separated by "---".`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 15,
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'plan-ceo-review-expansion-energy',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -270,7 +271,7 @@ Write your expansion proposals to ${planDir}/proposals.md with ONLY the proposal
|
||||
// Pass threshold: 4/5 on both axes (good — matches posture with minor weakness).
|
||||
expect(scores.axis_a).toBeGreaterThanOrEqual(4); // surface_framing
|
||||
expect(scores.axis_b).toBeGreaterThanOrEqual(4); // decision_preservation
|
||||
}, 600_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Plan Eng Review E2E ---
|
||||
@@ -348,7 +349,7 @@ Write your complete review directly to ${planDir}/review-output.md
|
||||
Focus on architecture, code quality, tests, and performance sections.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 15,
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'plan-eng-review',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -366,7 +367,7 @@ Focus on architecture, code quality, tests, and performance sections.`,
|
||||
const review = fs.readFileSync(reviewPath, 'utf-8');
|
||||
expect(review.length).toBeGreaterThan(200);
|
||||
}
|
||||
}, 420_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Plan-Eng-Review Test-Plan Artifact E2E ---
|
||||
@@ -476,7 +477,7 @@ Write your review to ${planDir}/review-output.md`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 25,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Glob', 'Grep'],
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'plan-eng-review-artifact',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -507,7 +508,7 @@ Write your review to ${planDir}/review-output.md`,
|
||||
if (newFiles.length === 0) {
|
||||
console.warn('SOFT FAIL: No test-plan artifact written — agent did not follow artifact instructions');
|
||||
}
|
||||
}, 420_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Office Hours Spec Review E2E ---
|
||||
@@ -559,7 +560,7 @@ Write your summary to ${ohDir}/spec-review-summary.md`,
|
||||
// failures wrote a correct summary on tool-turn 8 and hit the cap on
|
||||
// the closing text turn (error_max_turns at 9 turns, deterministic).
|
||||
maxTurns: 12,
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'office-hours-spec-review',
|
||||
runId,
|
||||
});
|
||||
@@ -575,7 +576,7 @@ Write your summary to ${ohDir}/spec-review-summary.md`,
|
||||
expect(summary).toMatch(/agent|subagent/);
|
||||
expect(summary).toMatch(/3.*iteration|iteration.*3|maximum.*3/);
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Plan CEO Review Benefits-From E2E ---
|
||||
@@ -619,7 +620,7 @@ Summarize what happens when no design doc is found — specifically:
|
||||
Write your summary to ${benefitsDir}/benefits-summary.md`,
|
||||
workingDirectory: benefitsDir,
|
||||
maxTurns: 8,
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'plan-ceo-review-benefits',
|
||||
runId,
|
||||
});
|
||||
@@ -634,7 +635,7 @@ Write your summary to ${benefitsDir}/benefits-summary.md`,
|
||||
expect(summary).toMatch(/office.hours/);
|
||||
expect(summary).toMatch(/design doc|no design/i);
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Plan Review Report E2E ---
|
||||
@@ -706,7 +707,7 @@ CRITICAL REQUIREMENT: plan.md IS the plan file for this review session. After co
|
||||
This review report at the bottom of the plan is the MOST IMPORTANT deliverable of this test.`,
|
||||
workingDirectory: planDir,
|
||||
maxTurns: 20,
|
||||
timeout: 360_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'plan-review-report',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -768,7 +769,7 @@ This review report at the bottom of the plan is the MOST IMPORTANT deliverable o
|
||||
).toBe(true);
|
||||
|
||||
console.log('Plan review report found at bottom of plan.md (ends with unresolved status)');
|
||||
}, 420_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Codex Offering E2E ---
|
||||
@@ -825,7 +826,7 @@ Summarize the Codex/${featureName} integration — answer these specific questio
|
||||
Write your summary to ${testDir}/${testName}-summary.md`,
|
||||
workingDirectory: testDir,
|
||||
maxTurns: 8,
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
});
|
||||
@@ -850,19 +851,19 @@ Write your summary to ${testDir}/${testName}-summary.md`,
|
||||
|
||||
testConcurrentIfSelected('codex-offered-office-hours', async () => {
|
||||
await checkCodexOffering('office-hours', 'codex-offered-office-hours', 'second opinion');
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testConcurrentIfSelected('codex-offered-ceo-review', async () => {
|
||||
await checkCodexOffering('plan-ceo-review', 'codex-offered-ceo-review', 'outside voice');
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testConcurrentIfSelected('codex-offered-design-review', async () => {
|
||||
await checkCodexOffering('plan-design-review', 'codex-offered-design-review', 'design outside voices');
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
testConcurrentIfSelected('codex-offered-eng-review', async () => {
|
||||
await checkCodexOffering('plan-eng-review', 'codex-offered-eng-review', 'outside voice');
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -40,6 +40,8 @@ const INLINE_REF = '29785978'; // last pre-Phase-1 commit (v1.69.1.0 bump)
|
||||
|
||||
function inlineSkill(): string {
|
||||
return execSync(`git show ${INLINE_REF}:plan-ceo-review/SKILL.md`, {
|
||||
// LIVE-REPO CWD: git show needs this repo's history to read the
|
||||
// pre-Phase-1 SKILL.md render at INLINE_REF.
|
||||
cwd: ROOT,
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import { outcomeJudge } from './helpers/llm-judge';
|
||||
import { judgePassed } from './helpers/eval-store';
|
||||
@@ -97,7 +98,7 @@ CRITICAL RULES:
|
||||
- The report MUST exist at ${reportPath} when you finish`,
|
||||
workingDirectory: testWorkDir,
|
||||
maxTurns: 50,
|
||||
timeout: 300_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: `qa-${label}`,
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -174,17 +175,17 @@ CRITICAL RULES:
|
||||
// B6: Static dashboard — broken link, disabled submit, overflow, missing alt, console error
|
||||
testConcurrentIfSelected('qa-b6-static', async () => {
|
||||
await runPlantedBugEval('qa-eval.html', 'qa-eval-ground-truth.json', 'b6-static');
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
// B7: SPA — broken route, stale state, async race, missing aria, console warning
|
||||
testConcurrentIfSelected('qa-b7-spa', async () => {
|
||||
await runPlantedBugEval('qa-eval-spa.html', 'qa-eval-spa-ground-truth.json', 'b7-spa');
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
// B8: Checkout — email regex, NaN total, CC overflow, missing required, stripe error
|
||||
testConcurrentIfSelected('qa-b8-checkout', async () => {
|
||||
await runPlantedBugEval('qa-eval-checkout.html', 'qa-eval-checkout-ground-truth.json', 'b8-checkout');
|
||||
}, 360_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled,
|
||||
@@ -54,7 +55,7 @@ Do NOT try to start a server or discover ports — the URL above is ready.
|
||||
Write your report to ${qaDir}/qa-reports/qa-report.md`,
|
||||
workingDirectory: qaDir,
|
||||
maxTurns: 35,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'qa-quick',
|
||||
runId,
|
||||
});
|
||||
@@ -69,7 +70,7 @@ Write your report to ${qaDir}/qa-reports/qa-report.md`,
|
||||
}
|
||||
// Accept error_max_turns — the agent doing thorough QA work is not a failure
|
||||
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- QA-Only E2E (report-only, no fixes) ---
|
||||
@@ -124,7 +125,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`,
|
||||
workingDirectory: qaOnlyDir,
|
||||
maxTurns: 40,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Glob'], // NO Edit — the critical guardrail
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'qa-only-no-fix',
|
||||
runId,
|
||||
});
|
||||
@@ -156,7 +157,7 @@ Write your report to ${qaOnlyDir}/qa-reports/qa-only-report.md`,
|
||||
(l: string) => l.trim() && !l.includes('.prompt-tmp') && !l.includes('.gstack/') && !l.includes('qa-reports/'),
|
||||
);
|
||||
expect(statusLines.filter((l: string) => l.startsWith(' M') || l.startsWith('M '))).toHaveLength(0);
|
||||
}, 240_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- QA Fix Loop E2E ---
|
||||
@@ -247,7 +248,7 @@ This is a test+fix loop: find bugs, fix them in the source code, commit each fix
|
||||
workingDirectory: qaFixDir,
|
||||
maxTurns: 40,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
|
||||
timeout: 420_000,
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'qa-fix-loop',
|
||||
runId,
|
||||
});
|
||||
@@ -271,7 +272,7 @@ This is a test+fix loop: find bugs, fix them in the source code, commit each fix
|
||||
// Verify Edit tool was used (agent actually modified source code)
|
||||
const editCalls = result.toolCalls.filter(tc => tc.tool === 'Edit');
|
||||
expect(editCalls.length).toBeGreaterThan(0);
|
||||
}, 480_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Test Bootstrap E2E ---
|
||||
@@ -384,7 +385,7 @@ Do NOT fix any bugs. Do NOT use AskUserQuestion — just pick vitest.`,
|
||||
workingDirectory: bsDir,
|
||||
maxTurns: 12,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'qa-bootstrap',
|
||||
runId,
|
||||
});
|
||||
@@ -405,7 +406,7 @@ Do NOT fix any bugs. Do NOT use AskUserQuestion — just pick vitest.`,
|
||||
console.log(`Test config: ${hasTestConfig}, Test file: ${hasTestFile}, TESTING.md: ${hasTestingMd}`);
|
||||
|
||||
try { fs.rmSync(bsDir, { recursive: true, force: true }); } catch {}
|
||||
}, 120_000);
|
||||
}, JUDGE_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
@@ -135,7 +136,7 @@ Write your retrospective to ${dir}/retro-output.md`,
|
||||
expect(wroteReport).toBe(true);
|
||||
const content = fs.readFileSync(retroPath, 'utf-8');
|
||||
expect(content.length).toBeGreaterThan(100);
|
||||
}, 480_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Retro E2E ---
|
||||
@@ -198,7 +199,7 @@ Write your retrospective report to ${retroDir}/retro-output.md
|
||||
Analyze the git history and produce the narrative report as described in the SKILL.md.`,
|
||||
workingDirectory: retroDir,
|
||||
maxTurns: 30,
|
||||
timeout: 300_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'retro',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
@@ -217,7 +218,7 @@ Analyze the git history and produce the narrative report as described in the SKI
|
||||
expect(wroteReport).toBe(true);
|
||||
const retro = fs.readFileSync(retroPath, 'utf-8');
|
||||
expect(retro.length).toBeGreaterThan(100);
|
||||
}, 420_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId, describeIfSelected, testConcurrentIfSelected,
|
||||
@@ -114,7 +115,7 @@ and apply it yourself against the diff (git diff main...HEAD).
|
||||
Write your findings to ${dir}/review-output.md`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 20,
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'review-army-migration-safety',
|
||||
runId,
|
||||
});
|
||||
@@ -135,7 +136,7 @@ Write your findings to ${dir}/review-output.md`,
|
||||
content.includes('column');
|
||||
expect(hasMigrationFinding).toBe(true);
|
||||
}
|
||||
}, 210_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Review Army: N+1 Performance ---
|
||||
@@ -179,7 +180,7 @@ For the specialist dispatch, read review-specialists/performance.md and apply it
|
||||
Write your findings to ${dir}/review-output.md`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 20,
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'review-army-perf-n-plus-one',
|
||||
runId,
|
||||
});
|
||||
@@ -201,7 +202,7 @@ Write your findings to ${dir}/review-output.md`,
|
||||
content.includes('loop');
|
||||
expect(hasN1Finding).toBe(true);
|
||||
}
|
||||
}, 210_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Review Army: Delivery Audit ---
|
||||
@@ -281,7 +282,7 @@ The email notification system should be classified as NOT DONE.
|
||||
Write your completion audit to ${dir}/review-output.md`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 15,
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'review-army-delivery-audit',
|
||||
runId,
|
||||
});
|
||||
@@ -305,7 +306,7 @@ Write your completion audit to ${dir}/review-output.md`,
|
||||
expect(hasNotDone).toBe(true);
|
||||
expect(mentionsEmail).toBe(true);
|
||||
}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Review Army: Quality Score ---
|
||||
@@ -356,7 +357,7 @@ Write your findings AND the computed quality score to ${dir}/review-output.md
|
||||
Include the line: "PR Quality Score: X/10" where X is the computed score.`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 15,
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'review-army-quality-score',
|
||||
runId,
|
||||
});
|
||||
@@ -374,7 +375,7 @@ Include the line: "PR Quality Score: X/10" where X is the computed score.`,
|
||||
content.match(/\d+\/10/);
|
||||
expect(hasScore).toBeTruthy();
|
||||
}
|
||||
}, 150_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Review Army: JSON Findings ---
|
||||
@@ -421,7 +422,7 @@ Output your findings as JSON objects, one per line, following the schema:
|
||||
Write ONLY JSON findings (no preamble) to ${dir}/findings.json`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 12,
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'review-army-json-findings',
|
||||
runId,
|
||||
});
|
||||
@@ -450,7 +451,7 @@ Write ONLY JSON findings (no preamble) to ${dir}/findings.json`,
|
||||
break; // One valid line is enough for the gate test
|
||||
}
|
||||
}
|
||||
}, 120_000);
|
||||
}, JUDGE_MS);
|
||||
});
|
||||
|
||||
// --- Review Army: Red Team (periodic) ---
|
||||
@@ -499,7 +500,7 @@ Write your red team findings to ${dir}/review-output.md
|
||||
Start the file with "RED TEAM REVIEW" on the first line.`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 20,
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'review-army-red-team',
|
||||
runId,
|
||||
});
|
||||
@@ -513,7 +514,7 @@ Start the file with "RED TEAM REVIEW" on the first line.`,
|
||||
const content = fs.readFileSync(outputPath, 'utf-8');
|
||||
expect(content.toLowerCase()).toMatch(/red team|adversarial/);
|
||||
}
|
||||
}, 210_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Review Army: Consensus (periodic) ---
|
||||
@@ -566,7 +567,7 @@ mark it as "MULTI-SPECIALIST CONFIRMED" with the confirming categories.
|
||||
Write findings to ${dir}/review-output.md`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 20,
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'review-army-consensus',
|
||||
runId,
|
||||
});
|
||||
@@ -585,7 +586,7 @@ Write findings to ${dir}/review-output.md`,
|
||||
content.includes('interpolat');
|
||||
expect(hasSqlFinding).toBe(true);
|
||||
}
|
||||
}, 210_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Review Army: Simplification specialist (activation) ---
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
@@ -65,7 +66,7 @@ Then run git diff against the detected base branch and write a brief review.
|
||||
Write your findings to ${dir}/review-output.md`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 15,
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'review-base-branch',
|
||||
runId,
|
||||
});
|
||||
@@ -84,7 +85,7 @@ Write your findings to ${dir}/review-output.md`,
|
||||
return cmd.includes('git diff');
|
||||
});
|
||||
expect(usedGitDiff).toBe(true);
|
||||
}, 120_000);
|
||||
}, JUDGE_MS);
|
||||
|
||||
testConcurrentIfSelected('ship-base-branch', async () => {
|
||||
const dir = path.join(baseBranchDir, 'ship-base');
|
||||
@@ -125,7 +126,7 @@ Write a summary to ${dir}/ship-preflight.md including:
|
||||
- The diff stat against the base branch`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 18,
|
||||
timeout: 150_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'ship-base-branch',
|
||||
runId,
|
||||
});
|
||||
@@ -155,7 +156,7 @@ Write a summary to ${dir}/ship-preflight.md including:
|
||||
return command.includes('git push') || command.includes('gh pr create');
|
||||
});
|
||||
expect(destructiveTools).toHaveLength(0);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Review Dashboard Via Attribution E2E ---
|
||||
@@ -280,7 +281,7 @@ Write the dashboard output to ${dashDir}/dashboard-output.md`,
|
||||
);
|
||||
// Ship dashboard should not gate when eng review is clear
|
||||
expect(gateQuestions).toHaveLength(0);
|
||||
}, 480_000);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled, selectedTests,
|
||||
@@ -66,7 +67,7 @@ Run /review on the current diff (git diff main...HEAD).
|
||||
Write your review findings to ${reviewDir}/review-output.md`,
|
||||
workingDirectory: reviewDir,
|
||||
maxTurns: 20,
|
||||
timeout: 180_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'review-sql-injection',
|
||||
runId,
|
||||
});
|
||||
@@ -89,7 +90,7 @@ Write your review findings to ${reviewDir}/review-output.md`,
|
||||
reviewContent.includes('unsanitized');
|
||||
expect(hasSqlContent).toBe(true);
|
||||
}
|
||||
}, 210_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Review: Enum completeness E2E ---
|
||||
@@ -144,7 +145,7 @@ Write your review findings to ${enumDir}/review-output.md
|
||||
The diff adds a new "returned" status to the Order model. Your job is to check if all consumers handle it.`,
|
||||
workingDirectory: enumDir,
|
||||
maxTurns: 15,
|
||||
timeout: 90_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'review-enum-completeness',
|
||||
runId,
|
||||
});
|
||||
@@ -164,7 +165,7 @@ The diff adds a new "returned" status to the Order model. Your job is to check i
|
||||
expect(mentionsReturned).toBe(true);
|
||||
expect(mentionsEnum || mentionsCritical).toBe(true);
|
||||
}
|
||||
}, 120_000);
|
||||
}, JUDGE_MS);
|
||||
});
|
||||
|
||||
// --- Review: Design review lite E2E ---
|
||||
@@ -229,7 +230,7 @@ Write your review findings to ${designDir}/review-output.md
|
||||
Important: The design checklist should catch issues like blacklisted fonts, small font sizes, outline:none, !important, AI slop patterns (purple gradients, generic hero copy, 3-column feature grid), etc.`,
|
||||
workingDirectory: designDir,
|
||||
maxTurns: 35,
|
||||
timeout: 240_000,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'review-design-lite',
|
||||
runId,
|
||||
});
|
||||
@@ -262,7 +263,7 @@ Important: The design checklist should catch issues like blacklisted fonts, smal
|
||||
console.log(`Design review detected ${detected}/7 planted issues`);
|
||||
expect(detected).toBeGreaterThanOrEqual(4);
|
||||
}
|
||||
}, 300_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// Base branch detection tests for review/ship + the Review Dashboard Via
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId, evalsEnabled,
|
||||
@@ -162,11 +163,12 @@ IMPORTANT:
|
||||
Replace any references to ~/.claude/skills/gstack/bin/ with ./bin/ when running commands.
|
||||
- Do NOT use AskUserQuestion.
|
||||
- Just run the preamble bash block and report what you see.
|
||||
- Look for "RECENT ARTIFACTS" and "LAST_SESSION" in the output.`,
|
||||
- Look for "RECENT ARTIFACTS" and "LAST_SESSION" in the output.
|
||||
- In your final message, quote VERBATIM (copy exactly, do not paraphrase) any output lines containing "RECENT ARTIFACTS" or "LAST_SESSION".`,
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 10,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-recovery-artifacts',
|
||||
runId,
|
||||
});
|
||||
@@ -193,7 +195,7 @@ IMPORTANT:
|
||||
expect(foundCount).toBeGreaterThanOrEqual(1);
|
||||
|
||||
console.log(`Context recovery: artifacts=${foundArtifacts}, lastSession=${foundLastSession}, timeline=${foundTimeline}`);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// --- Test 3: /context-save writes a file ---
|
||||
// Hand-feed the save section of context-save/SKILL.md to claude -p and verify
|
||||
@@ -231,7 +233,7 @@ IMPORTANT:
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 10,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-save-writes-file',
|
||||
runId,
|
||||
});
|
||||
@@ -264,7 +266,7 @@ IMPORTANT:
|
||||
expect(hasYamlFrontmatter).toBe(true);
|
||||
|
||||
console.log(`context-save: ${files.length} files created, YAML frontmatter: ${hasYamlFrontmatter}, branch: ${hasBranch}`);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// --- Test 4: /context-restore loads the newest file across branches ---
|
||||
// Seed two saved-context files with different YYYYMMDD-HHMMSS prefixes and
|
||||
@@ -272,7 +274,17 @@ IMPORTANT:
|
||||
// claude -p. Verify the agent identifies the newer file (by filename prefix)
|
||||
// and presents its content, regardless of the current branch.
|
||||
testConcurrentIfSelected('context-restore-loads-latest', async () => {
|
||||
const projectDir = path.join(gstackHome, 'projects', slug);
|
||||
// PRIVATE home for this test: the suite runs concurrently, and the shared
|
||||
// gstackHome's checkpoints dir also receives the context-save test's
|
||||
// freshly-written checkpoint (a 2026-08-29 filename prefix — always the
|
||||
// "newest"). In CI, save completed before this test's agent listed the
|
||||
// dir, so the agent CORRECTLY restored the sibling's checkpoint and the
|
||||
// assertions failed; locally the ordering happened to run restore first.
|
||||
// An isolated home makes the fixture set closed regardless of ordering
|
||||
// (the agent may derive the dir from GSTACK_HOME/projects/<slug>, so the
|
||||
// whole home moves, not just the checkpoint path we hand it).
|
||||
const restoreHome = path.join(workDir, '.gstack-restore-home');
|
||||
const projectDir = path.join(restoreHome, 'projects', slug);
|
||||
const checkpointDir = path.join(projectDir, 'checkpoints');
|
||||
fs.mkdirSync(checkpointDir, { recursive: true });
|
||||
|
||||
@@ -331,16 +343,19 @@ This is the newest saved context. Cross-branch restore should load THIS file.
|
||||
${restoreSection.slice(0, 2500)}
|
||||
|
||||
IMPORTANT:
|
||||
- Use GSTACK_HOME="${gstackHome}" as an environment variable when running bin scripts.
|
||||
- Use GSTACK_HOME="${restoreHome}" as an environment variable when running bin scripts.
|
||||
- The bin scripts are at ./bin/ (relative to this directory), not at ~/.claude/skills/gstack/bin/.
|
||||
- Look in ${checkpointDir} for saved context files.
|
||||
- Current branch is "main" — do NOT filter by current branch. Load across all branches.
|
||||
- The newest file by YYYYMMDD-HHMMSS prefix is the canonical "most recent". Filesystem mtime has been scrambled — do not use it.
|
||||
- Do NOT use AskUserQuestion. Just present the content of the newest file.`,
|
||||
- Do NOT use AskUserQuestion. Just present the content of the newest file.
|
||||
- Your final message MUST end with these two lines (they are machine-checked — copy them exactly, do not paraphrase):
|
||||
1. The newest file's "## Working on:" heading line, VERBATIM as it appears in that file.
|
||||
2. A literal marker line: RESTORED: <filename of the newest file>`,
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 8,
|
||||
allowedTools: ['Bash', 'Read', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'context-restore-loads-latest',
|
||||
runId,
|
||||
});
|
||||
@@ -348,8 +363,34 @@ IMPORTANT:
|
||||
logCost('context-restore', result);
|
||||
|
||||
const output = result.output ?? '';
|
||||
const loadedNewer = output.includes('newer wintermute work') || output.includes('wintermute integration');
|
||||
const loadedOlder = output.includes('old work') && !output.includes('newer');
|
||||
|
||||
// Evidence-based checks. CI receipts showed the agent finding + reading the
|
||||
// RIGHT file, then paraphrasing the final message ("the most recent context
|
||||
// is from branch-b...") — exact-substring checks over stochastic prose
|
||||
// flaked. Three signal classes, strongest first:
|
||||
// 1. Machine-checkable output contract (prompt demands the verbatim
|
||||
// "## Working on:" heading + a "RESTORED: <filename>" marker line).
|
||||
// 2. Lenient content echo (legacy): distinctive newer-file phrases.
|
||||
// 3. Tool-call corroboration: a tool call whose INPUT names the newer
|
||||
// file. Corroboration only — if the agent also read the OLDER file,
|
||||
// tool evidence is void and the final output must present the newer.
|
||||
const newerFileName = '20260202-130000-newer-wintermute-work';
|
||||
const olderFileName = '20260101-120000-old-work';
|
||||
const newerMarker = new RegExp(`RESTORED:.*${newerFileName}`, 'i').test(output);
|
||||
const olderMarker = new RegExp(`RESTORED:.*${olderFileName}`, 'i').test(output);
|
||||
const newerContent = output.includes('newer wintermute work') || output.includes('wintermute integration');
|
||||
const outputPresentsNewer = newerMarker || newerContent;
|
||||
|
||||
const toolInputs = result.toolCalls.map(tc => JSON.stringify(tc.input ?? {}));
|
||||
const toolReadNewer = toolInputs.some(input => input.includes(newerFileName));
|
||||
const toolReadOlder = toolInputs.some(input => input.includes(olderFileName));
|
||||
|
||||
// Presenting the OLDER file fails: an explicit RESTORED marker naming it,
|
||||
// or older-file content with no newer-file presentation alongside.
|
||||
const loadedOlder = olderMarker || (output.includes('old work') && !outputPresentsNewer);
|
||||
// Tool evidence counts only when the older file was never read: a run that
|
||||
// reads BOTH files must present the NEWER one in the final output to pass.
|
||||
const loadedNewer = outputPresentsNewer || (toolReadNewer && !toolReadOlder);
|
||||
const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);
|
||||
|
||||
recordE2E(evalCollector, 'context-restore loads latest', 'Session Intelligence E2E', result, {
|
||||
@@ -360,6 +401,6 @@ IMPORTANT:
|
||||
expect(loadedNewer).toBe(true);
|
||||
expect(loadedOlder).toBe(false);
|
||||
|
||||
console.log(`context-restore: loadedNewer=${loadedNewer}, loadedOlder=${loadedOlder}`);
|
||||
}, 180_000);
|
||||
console.log(`context-restore: loadedNewer=${loadedNewer} (marker=${newerMarker}, content=${newerContent}, toolNewer=${toolReadNewer}, toolOlder=${toolReadOlder}), loadedOlder=${loadedOlder}`);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user