mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-10 23:19:09 +02:00
Merge remote-tracking branch 'origin/main' into garrytan/binding-polarity-wave
This commit is contained in:
@@ -45,7 +45,7 @@ function uuid(): string {
|
||||
return `00000000-0000-0000-0000-${String(++uuidCounter).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
function systemInit(model = 'claude-opus-4-7', version = '2.1.117'): SDKMessage {
|
||||
function systemInit(model = 'claude-sonnet-4-6', version = '2.1.117'): SDKMessage {
|
||||
return {
|
||||
type: 'system',
|
||||
subtype: 'init',
|
||||
@@ -77,7 +77,7 @@ function assistantTurn(
|
||||
id: 'msg_' + uuid(),
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model: 'claude-opus-4-7',
|
||||
model: 'claude-sonnet-4-6',
|
||||
content: blocks.map((b) => ({ ...b })),
|
||||
stop_reason: 'end_turn',
|
||||
stop_sequence: null,
|
||||
@@ -259,7 +259,7 @@ describe('runAgentSdkTest — happy path', () => {
|
||||
expect(result.turnsUsed).toBe(2);
|
||||
expect(result.costUsd).toBe(0.05);
|
||||
expect(result.sdkClaudeCodeVersion).toBe('2.1.117');
|
||||
expect(result.model).toBe('claude-opus-4-7');
|
||||
expect(result.model).toBe('claude-sonnet-4-6');
|
||||
expect(result.firstResponseMs).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
@@ -699,7 +699,7 @@ describe('toSkillTestResult', () => {
|
||||
expect(s.output).toBe('hi');
|
||||
expect(s.costEstimate.estimatedCost).toBe(0.02);
|
||||
expect(s.costEstimate.turnsUsed).toBe(1);
|
||||
expect(s.model).toBe('claude-opus-4-7');
|
||||
expect(s.model).toBe('claude-sonnet-4-6');
|
||||
expect(s.firstResponseMs).toBeNumber();
|
||||
expect(s.maxInterTurnMs).toBeNumber();
|
||||
expect(s.transcript).toBeArray();
|
||||
@@ -715,7 +715,7 @@ describe('validateFixtures', () => {
|
||||
return {
|
||||
id: 'test-fixture',
|
||||
overlayPath: 'model-overlays/opus-4-7.md',
|
||||
model: 'claude-opus-4-7',
|
||||
model: 'claude-sonnet-4-6',
|
||||
trials: 10,
|
||||
setupWorkspace: () => {},
|
||||
userPrompt: 'go',
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
/**
|
||||
* Regression pins for the preflight-dedup seam (test/helpers/anthropic-preflight.ts).
|
||||
*
|
||||
* The preflight runs at MODULE LOAD in every paid test file that imports
|
||||
* e2e-helpers, so a broken skip-check either brings back ~30 paid pings per
|
||||
* sharded run or — worse — skips the fail-fast everywhere. Both directions
|
||||
* are pinned here with an injected spawn; no real claude call is made.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { preflightAnthropicApi } from './helpers/anthropic-preflight';
|
||||
|
||||
type SpawnCall = { command: string; args: string[] };
|
||||
|
||||
function fakeSpawn(stdout: string, calls: SpawnCall[]) {
|
||||
return ((command: string, args: string[]) => {
|
||||
calls.push({ command, args });
|
||||
return { stdout: Buffer.from(stdout), stderr: Buffer.from(''), status: 0 } as ReturnType<
|
||||
typeof import('child_process').spawnSync
|
||||
>;
|
||||
}) as typeof import('child_process').spawnSync;
|
||||
}
|
||||
|
||||
describe('preflightAnthropicApi', () => {
|
||||
test('EVALS_PREFLIGHT_OK=1 skips the ping entirely (sharded-child path)', () => {
|
||||
const calls: SpawnCall[] = [];
|
||||
const result = preflightAnthropicApi({ EVALS_PREFLIGHT_OK: '1' }, fakeSpawn('never read', calls));
|
||||
expect(result).toBe('skipped');
|
||||
expect(calls.length).toBe(0);
|
||||
});
|
||||
|
||||
test('without the flag, pings exactly once and passes on healthy output', () => {
|
||||
const calls: SpawnCall[] = [];
|
||||
const result = preflightAnthropicApi({}, fakeSpawn('{"type":"result"}', calls));
|
||||
expect(result).toBe('ok');
|
||||
expect(calls.length).toBe(1);
|
||||
expect(calls[0].args.join(' ')).toContain('claude -p');
|
||||
});
|
||||
|
||||
test('unreachable API throws (fail-fast before any shard spawns)', () => {
|
||||
const calls: SpawnCall[] = [];
|
||||
expect(() =>
|
||||
preflightAnthropicApi({}, fakeSpawn('error: ConnectionRefused connecting to api', calls)),
|
||||
).toThrow(/Anthropic API unreachable/);
|
||||
});
|
||||
|
||||
test('a truthy-but-not-"1" flag still pings (no accidental widening)', () => {
|
||||
const calls: SpawnCall[] = [];
|
||||
const result = preflightAnthropicApi({ EVALS_PREFLIGHT_OK: 'true' }, fakeSpawn('ok', calls));
|
||||
expect(result).toBe('ok');
|
||||
expect(calls.length).toBe(1);
|
||||
});
|
||||
|
||||
// Failure modes that guarantee every shard fails too must fail the
|
||||
// preflight — previously a missing binary, a timeout kill, and exit 127
|
||||
// all returned 'ok' and the fleet burned its own discovery of the outage.
|
||||
const shapedSpawn = (shape: Partial<ReturnType<typeof import('child_process').spawnSync>>) =>
|
||||
((() => ({ stdout: Buffer.from(''), stderr: Buffer.from(''), status: 0, ...shape })) as unknown as
|
||||
typeof import('child_process').spawnSync);
|
||||
|
||||
test('spawn error (unlaunchable shell/binary) throws', () => {
|
||||
expect(() =>
|
||||
preflightAnthropicApi({}, shapedSpawn({ error: new Error('spawn sh ENOENT') })),
|
||||
).toThrow(/could not run/);
|
||||
});
|
||||
|
||||
test('timeout kill (signal set) throws', () => {
|
||||
expect(() =>
|
||||
preflightAnthropicApi({}, shapedSpawn({ signal: 'SIGTERM' })),
|
||||
).toThrow(/timed out/);
|
||||
});
|
||||
|
||||
test('exit 127 (claude not found) throws', () => {
|
||||
expect(() =>
|
||||
preflightAnthropicApi({}, shapedSpawn({ status: 127 })),
|
||||
).toThrow(/not found on PATH/);
|
||||
});
|
||||
|
||||
test('other non-zero exits stay fail-open (a flaky preflight must not block a runnable suite)', () => {
|
||||
expect(preflightAnthropicApi({}, shapedSpawn({ status: 1, stdout: Buffer.from('transient') }))).toBe('ok');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* getChangedFiles union semantics: committed + staged + unstaged + untracked.
|
||||
* Free (no API calls), runs with `bun test`.
|
||||
*
|
||||
* Change-set 3 of the eval-selection work: an agent that edits files and
|
||||
* runs evals BEFORE committing used to get an empty committed-diff → run-all
|
||||
* → full paid suite. getChangedFiles now unions the committed diff with the
|
||||
* working-tree diff and untracked files, and FAILS CLOSED (throws, naming
|
||||
* EVALS_ALL=1) on any git error instead of silently returning [].
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
|
||||
import { getChangedFiles } from './helpers/touchfiles';
|
||||
|
||||
describe('getChangedFiles union', () => {
|
||||
let repo: string;
|
||||
|
||||
const git = (args: string[]) => {
|
||||
const result = spawnSync(
|
||||
'git',
|
||||
['-c', 'user.email=test@test', '-c', 'user.name=test', '-c', 'commit.gpgsign=false', ...args],
|
||||
{ cwd: repo, stdio: 'pipe', timeout: 10000 },
|
||||
);
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`git ${args.join(' ')} failed: ${result.stderr?.toString()}`);
|
||||
}
|
||||
};
|
||||
|
||||
const write = (rel: string, content: string) => {
|
||||
const filePath = path.join(repo, rel);
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
fs.writeFileSync(filePath, content);
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
repo = fs.mkdtempSync(path.join(os.tmpdir(), 'changed-files-union-'));
|
||||
git(['init', '-q']);
|
||||
write('a.txt', 'a\n');
|
||||
write('b.txt', 'b\n');
|
||||
git(['add', 'a.txt', 'b.txt']);
|
||||
git(['commit', '-q', '-m', 'base']);
|
||||
git(['tag', 'base']);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(repo, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('committed-only change', () => {
|
||||
write('a.txt', 'a2\n');
|
||||
git(['add', 'a.txt']);
|
||||
git(['commit', '-q', '-m', 'change a']);
|
||||
expect(getChangedFiles('base', repo)).toEqual(['a.txt']);
|
||||
});
|
||||
|
||||
test('staged-only change', () => {
|
||||
write('a.txt', 'a2\n');
|
||||
git(['add', 'a.txt']);
|
||||
expect(getChangedFiles('base', repo)).toEqual(['a.txt']);
|
||||
});
|
||||
|
||||
test('unstaged-only change', () => {
|
||||
write('b.txt', 'b2\n');
|
||||
expect(getChangedFiles('base', repo)).toEqual(['b.txt']);
|
||||
});
|
||||
|
||||
test('untracked-only file', () => {
|
||||
write('new-dir/new.txt', 'new\n');
|
||||
expect(getChangedFiles('base', repo)).toEqual(['new-dir/new.txt']);
|
||||
});
|
||||
|
||||
test('mixed sources — each file exactly once', () => {
|
||||
// committed change to a.txt...
|
||||
write('a.txt', 'a2\n');
|
||||
git(['add', 'a.txt']);
|
||||
git(['commit', '-q', '-m', 'change a']);
|
||||
// ...PLUS an unstaged edit to the same file (dedupe check),
|
||||
write('a.txt', 'a3\n');
|
||||
// a staged edit to b.txt,
|
||||
write('b.txt', 'b2\n');
|
||||
git(['add', 'b.txt']);
|
||||
// and an untracked file.
|
||||
write('c.txt', 'c\n');
|
||||
|
||||
const result = getChangedFiles('base', repo);
|
||||
expect(result.sort()).toEqual(['a.txt', 'b.txt', 'c.txt']);
|
||||
expect(result.filter(f => f === 'a.txt').length).toBe(1); // deduped
|
||||
});
|
||||
|
||||
test('clean tree → empty union (run-all semantics preserved by callers)', () => {
|
||||
expect(getChangedFiles('base', repo)).toEqual([]);
|
||||
});
|
||||
|
||||
test('non-repo cwd → throws naming EVALS_ALL', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'changed-files-nonrepo-'));
|
||||
try {
|
||||
expect(() => getChangedFiles('main', dir)).toThrow(/EVALS_ALL=1/);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('missing base ref → throws naming EVALS_ALL and the failing command', () => {
|
||||
expect(() => getChangedFiles('no-such-ref', repo)).toThrow(/EVALS_ALL=1/);
|
||||
expect(() => getChangedFiles('no-such-ref', repo)).toThrow(/diff --name-only no-such-ref\.\.\.HEAD/);
|
||||
});
|
||||
|
||||
test('non-ASCII filenames come back as raw UTF-8, not C-escaped (core.quotePath=false)', () => {
|
||||
const name = 'résumé-fixture.md';
|
||||
fs.writeFileSync(path.join(repo, name), 'x');
|
||||
try {
|
||||
const files = getChangedFiles('base', repo);
|
||||
expect(files).toContain(name);
|
||||
expect(files.every((f) => !f.includes('\\303'))).toBe(true);
|
||||
} finally {
|
||||
fs.rmSync(path.join(repo, name), { force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('injected spawn failure → throws with stderr in the message', () => {
|
||||
const failingSpawn = ((_cmd: string, args: string[]) => ({
|
||||
status: 128,
|
||||
error: undefined,
|
||||
stdout: Buffer.from(''),
|
||||
stderr: Buffer.from(`fatal: injected failure for ${args[0]}`),
|
||||
})) as unknown as typeof spawnSync;
|
||||
|
||||
let message = '';
|
||||
try {
|
||||
getChangedFiles('base', repo, failingSpawn);
|
||||
} catch (err) {
|
||||
message = (err as Error).message;
|
||||
}
|
||||
expect(message).toContain('EVALS_ALL=1');
|
||||
expect(message).toContain('injected failure');
|
||||
expect(message).toContain('exit 128');
|
||||
});
|
||||
|
||||
test('injected spawn error object (git binary missing) → throws', () => {
|
||||
const errorSpawn = (() => ({
|
||||
status: null,
|
||||
error: new Error('spawn git ENOENT'),
|
||||
stdout: Buffer.from(''),
|
||||
stderr: Buffer.from(''),
|
||||
})) as unknown as typeof spawnSync;
|
||||
|
||||
let message = '';
|
||||
try {
|
||||
getChangedFiles('base', repo, errorSpawn);
|
||||
} catch (err) {
|
||||
message = (err as Error).message;
|
||||
}
|
||||
expect(message).toContain('EVALS_ALL=1');
|
||||
expect(message).toContain('spawn git ENOENT');
|
||||
expect(message).toContain('spawn-error');
|
||||
});
|
||||
|
||||
test('untracked path with spaces (git quotes it) is unquoted', () => {
|
||||
write('has space.txt', 'x\n');
|
||||
expect(getChangedFiles('base', repo)).toEqual(['has space.txt']);
|
||||
});
|
||||
});
|
||||
+12
-3
@@ -3,7 +3,7 @@
|
||||
*
|
||||
* Spawns `codex exec` with skills installed in a temp HOME, parses JSONL
|
||||
* output, and validates structured results. Follows the same pattern as
|
||||
* skill-e2e.test.ts but adapted for Codex CLI.
|
||||
* the skill-e2e-*.test.ts suites but adapted for Codex CLI.
|
||||
*
|
||||
* Prerequisites:
|
||||
* - `codex` binary installed (npm install -g @openai/codex)
|
||||
@@ -16,6 +16,7 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { runCodexSkill, parseCodexJSONL, installSkillToTempHome } from './helpers/codex-session-runner';
|
||||
import type { CodexResult } from './helpers/codex-session-runner';
|
||||
import { CODEX_REVIEW_E2E_SECTIONS } from './helpers/skill-fixture';
|
||||
import { EvalCollector } from './helpers/eval-store';
|
||||
import type { EvalTestEntry } from './helpers/eval-store';
|
||||
import { selectTests, detectBaseBranch, getChangedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES } from './helpers/touchfiles';
|
||||
@@ -139,7 +140,11 @@ describeCodex('Codex E2E', () => {
|
||||
});
|
||||
|
||||
testIfSelected('codex-discover-skill', async () => {
|
||||
// Install gstack-review skill to a temp HOME and ask Codex to list skills
|
||||
// Install gstack-review skill to a temp HOME and ask Codex to list skills.
|
||||
// Deliberately installs the FULL generated SKILL.md (no `sections`): this
|
||||
// test's purpose is to prove the real artifact loads under Codex — the
|
||||
// stderr assertions below ('invalid' / 'Skipped loading') would be
|
||||
// meaningless against an extracted fixture.
|
||||
const skillDir = path.join(testWorktree, '.agents', 'skills', 'gstack-review');
|
||||
|
||||
const result = await runCodexSkill({
|
||||
@@ -172,7 +177,10 @@ describeCodex('Codex E2E', () => {
|
||||
// code review, and produce structured review output with findings/issues.
|
||||
// Accepts Codex timeout (exit 124/137) as non-failure since that's a CLI perf issue.
|
||||
testIfSelected('codex-review-findings', async () => {
|
||||
// Install gstack-review skill and ask Codex to review the worktree
|
||||
// Install gstack-review and ask Codex to review the worktree. The skill
|
||||
// fixture is EXTRACTED to the core review-workflow sections — the full
|
||||
// Codex host variant is ~1460 lines and this test only exercises the
|
||||
// diff-review flow (CLAUDE.md: "E2E test fixtures: extract, don't copy").
|
||||
const skillDir = path.join(testWorktree, '.agents', 'skills', 'gstack-review');
|
||||
|
||||
const result = await runCodexSkill({
|
||||
@@ -181,6 +189,7 @@ describeCodex('Codex E2E', () => {
|
||||
timeoutMs: 540_000,
|
||||
cwd: testWorktree,
|
||||
skillName: 'gstack-review',
|
||||
sections: CODEX_REVIEW_E2E_SECTIONS,
|
||||
});
|
||||
|
||||
logCodexCost('codex-review-findings', result);
|
||||
|
||||
@@ -90,8 +90,12 @@ describe("parseIntFlag contract (#2032, codex 17a-c)", () => {
|
||||
|
||||
describe("normalizeIntFlag CLI wrapper (exit-1 semantics)", () => {
|
||||
function runWrapper(rawExpr: string, specExpr: string): { status: number; stderr: string } {
|
||||
// Forward slashes: a raw Windows ROOT embeds backslashes into the eval
|
||||
// string where they act as ESCAPES ("D:\\a\\gstack" imports as
|
||||
// "D:agstack" — first Windows lane run). Import specifiers accept
|
||||
// forward slashes on every platform.
|
||||
const script = `
|
||||
import { normalizeIntFlag } from "${ROOT}/design/src/flag-utils";
|
||||
import { normalizeIntFlag } from "${ROOT.replaceAll('\\', '/')}/design/src/flag-utils";
|
||||
const v = normalizeIntFlag(${rawExpr}, ${specExpr});
|
||||
console.log("VALUE:" + v);
|
||||
`;
|
||||
|
||||
@@ -20,6 +20,8 @@ import { describe, test, expect } from 'bun:test';
|
||||
import { readdirSync, readFileSync } from 'fs';
|
||||
import * as path from 'path';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles';
|
||||
import { isPaidTestFile } from './helpers/paid-test-set';
|
||||
import { knownTestNamesInSource, PARENT_MAPPER_TEST_NAMES } from '../scripts/test-paid-shards';
|
||||
|
||||
const TEST_DIR = import.meta.dir;
|
||||
// Both quote styles — a mechanical refactor to double quotes must not
|
||||
@@ -95,4 +97,51 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () =>
|
||||
|
||||
expect(misaligned).toEqual([]);
|
||||
});
|
||||
|
||||
// HARD invariant (C6): the paid sharded runner skips a skill-e2e shard when
|
||||
// none of the file's MAPPED test names (E2E map keys quoted in its source,
|
||||
// union E2E map keys whose dep list registers the file) are diff-selected.
|
||||
// A skill-e2e file the mapper cannot see at all is only safe if it provably
|
||||
// opts out of name-based selection: it must not touch the e2e-helpers
|
||||
// selection surface (describeIfSelected / runSkillTest / selectedTests) AND
|
||||
// it must carry an explicit whole-file EVALS_TIER self-gate (the child-side
|
||||
// gate that makes the parent's fail-open keep semantically correct).
|
||||
//
|
||||
// Anything else is an invisible-test-names hole: the parent could drop a
|
||||
// shard whose child would have run real work. Fix by either quoting the
|
||||
// test's E2E map key as a string literal in the file, or adding the file's
|
||||
// path to its key's dep list in test/helpers/touchfiles-data.ts.
|
||||
test('every paid skill-e2e file is visible to the parent diff mapper (or provably fail-open-safe)', () => {
|
||||
const invisible: string[] = [];
|
||||
|
||||
for (const file of testFiles) {
|
||||
const repoPath = `test/${file}`;
|
||||
if (!isPaidTestFile(repoPath)) continue;
|
||||
const content = readFileSync(path.join(TEST_DIR, file), 'utf-8');
|
||||
|
||||
const quoted = knownTestNamesInSource(content, PARENT_MAPPER_TEST_NAMES);
|
||||
const registered = Object.keys(E2E_TOUCHFILES).filter((k) => E2E_TOUCHFILES[k].includes(repoPath));
|
||||
if (quoted.length + registered.length > 0) continue; // parent-mappable
|
||||
|
||||
const usesNameSelection = /\b(describeIfSelected|runSkillTest|selectedTests)\b/.test(content);
|
||||
// Both self-gate shapes count: the raw predicate and the consolidated
|
||||
// helper (test/helpers/e2e-gate.ts documents this file as a consumer
|
||||
// that must recognize describeE2ETier/e2eTierEnabled).
|
||||
const selfGated = /EVALS_TIER\s*===\s*['"](gate|periodic)['"]/.test(content)
|
||||
|| /\b(?:describeE2ETier|e2eTierEnabled)\(\s*['"](gate|periodic)['"]/.test(content);
|
||||
if (!usesNameSelection && selfGated) continue; // fail-open-safe standalone
|
||||
|
||||
invisible.push(
|
||||
`${repoPath}: invisible to the parent diff mapper — no E2E map key quoted in the file, `
|
||||
+ 'not registered in any E2E_TOUCHFILES dep list, and it '
|
||||
+ (usesNameSelection
|
||||
? 'uses name-based selection (describeIfSelected/runSkillTest/selectedTests)'
|
||||
: 'has no whole-file EVALS_TIER self-gate')
|
||||
+ '. Quote the test\'s E2E map key as a string literal, or add this file path to its '
|
||||
+ 'key\'s dep list in test/helpers/touchfiles-data.ts.',
|
||||
);
|
||||
}
|
||||
|
||||
expect(invisible).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -78,6 +78,9 @@ const MODULE_SINKS = [
|
||||
// missing file must fail loudly (a rename/move that drops its receipt wiring
|
||||
// is exactly what this pins), not silently soften the assertion.
|
||||
'lib/context-bill.ts',
|
||||
// supabase-provision engine (bin/gstack-gbrain-supabase-provision is a thin
|
||||
// bun-shebang entry over this module; the receipt lives at the api-call layer).
|
||||
'lib/gbrain-supabase-provision.ts',
|
||||
];
|
||||
|
||||
/** Shell sinks: must source the shared lib; every network op receipted. */
|
||||
@@ -88,7 +91,6 @@ const SHELL_SINKS = [
|
||||
'bin/gstack-gbrain-mcp-verify',
|
||||
'bin/gstack-security-dashboard',
|
||||
'bin/gstack-community-dashboard',
|
||||
'bin/gstack-gbrain-supabase-provision',
|
||||
'bin/gstack-artifacts-init',
|
||||
'bin/gstack-brain-restore',
|
||||
'bin/gstack-session-update',
|
||||
@@ -339,9 +341,15 @@ describe('egress receipt wiring tripwire', () => {
|
||||
// dashboards (open).
|
||||
expect(read('bin/gstack-security-dashboard')).toMatch(/_receipted_curl open security-dashboard/);
|
||||
expect(read('bin/gstack-community-dashboard')).toMatch(/_receipted_curl open community-dashboard/);
|
||||
// mcp-verify + provision (closed).
|
||||
// mcp-verify (closed).
|
||||
expect(read('bin/gstack-gbrain-mcp-verify')).toMatch(/_receipted_curl closed gbrain-mcp-verify/);
|
||||
expect(read('bin/gstack-gbrain-supabase-provision')).toMatch(/_receipted_curl closed supabase-provision/);
|
||||
// supabase-provision (closed): TS module — the receipt is written before
|
||||
// the fetch, and a receipt failure refuses the send (fail-closed, exit 8).
|
||||
const provision = read('lib/gbrain-supabase-provision.ts');
|
||||
expect(provision).toMatch(/sink:\s*['"]supabase-provision['"]/);
|
||||
expect(provision).toContain('fail-closed');
|
||||
expect(provision.indexOf('writeReceipt(')).toBeGreaterThan(0);
|
||||
expect(provision.indexOf('writeReceipt(')).toBeLessThan(provision.indexOf('ctx.fetchImpl('));
|
||||
// design (open): the wrapper catches receipt errors and proceeds.
|
||||
const rf = read('design/src/receipted-fetch.ts');
|
||||
expect(rf).toContain('fail-open');
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Detach-timeout floor — free, gate-tier tripwire.
|
||||
*
|
||||
* The eval:bg:gate / eval:bg:periodic scripts wrap the sharded paid runner in
|
||||
* bin/gstack-detach with a hard --timeout. If that number dips below the
|
||||
* runner's worst-case wall clock — ceil(shards / jobs) × shard timeout — the
|
||||
* watchdog kills a healthy run mid-flight and the tail shards report
|
||||
* never-started: paid truncation by configuration. That nearly shipped once
|
||||
* (a review pass proposed 10800s against a 19,800s gate worst case), so the
|
||||
* bound is enforced here against the LIVE shard census instead of a comment
|
||||
* snapshot that goes stale every time a paid test file is added.
|
||||
*
|
||||
* If this test fails you have two honest options: raise the --timeout in the
|
||||
* package.json script it names, or reduce the tier's worst case (split fewer
|
||||
* files per shard, raise DEFAULT_JOBS after verifying API rate headroom).
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import {
|
||||
collectPaidTestFiles,
|
||||
selectPaidTestFiles,
|
||||
DEFAULT_JOBS,
|
||||
DEFAULT_SHARD_TIMEOUT_MS,
|
||||
type PaidTier,
|
||||
} from '../scripts/test-paid-shards';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
// 5% margin over the theoretical bound: detach setup, lock wait, aggregation.
|
||||
const MARGIN = 1.05;
|
||||
|
||||
function detachTimeoutSeconds(scriptName: string): number {
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(ROOT, 'package.json'), 'utf-8'));
|
||||
const script: string | undefined = pkg.scripts?.[scriptName];
|
||||
expect(script, `package.json is missing the "${scriptName}" script`).toBeTruthy();
|
||||
const m = script!.match(/--timeout\s+(\d+)/);
|
||||
expect(m, `"${scriptName}" has no gstack-detach --timeout flag`).toBeTruthy();
|
||||
return parseInt(m![1], 10);
|
||||
}
|
||||
|
||||
function worstCaseSeconds(tier: PaidTier): number {
|
||||
const shards = selectPaidTestFiles(collectPaidTestFiles(), tier).selected.length;
|
||||
expect(shards).toBeGreaterThan(0);
|
||||
return Math.ceil(shards / DEFAULT_JOBS) * (DEFAULT_SHARD_TIMEOUT_MS / 1000);
|
||||
}
|
||||
|
||||
describe('eval:bg detach timeouts cover the sharded runner worst case', () => {
|
||||
for (const [tier, script] of [
|
||||
['gate', 'eval:bg:gate'],
|
||||
['periodic', 'eval:bg:periodic'],
|
||||
] as Array<[PaidTier, string]>) {
|
||||
test(`${script} >= ceil(${tier} shards / jobs) x shard timeout x ${MARGIN}`, () => {
|
||||
const floor = Math.ceil(worstCaseSeconds(tier) * MARGIN);
|
||||
const configured = detachTimeoutSeconds(script);
|
||||
if (configured < floor) {
|
||||
throw new Error(
|
||||
`${script} --timeout ${configured}s is below the ${tier} tier's worst-case ` +
|
||||
`wall clock of ${floor}s (ceil(shards/${DEFAULT_JOBS} jobs) x ` +
|
||||
`${DEFAULT_SHARD_TIMEOUT_MS / 1000}s shard timeout x ${MARGIN} margin). ` +
|
||||
`An undersized detach watchdog kills healthy runs mid-flight and the tail ` +
|
||||
`shards report never-started. Raise the --timeout in package.json or reduce ` +
|
||||
`the tier's worst case.`,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -13,7 +13,9 @@ describe("resolveEvalModel", () => {
|
||||
expect(resolveEvalModel("distill", null, { GSTACK_EVAL_MODEL: "g" } as never)).toBe("g");
|
||||
});
|
||||
test("defaults per kind", () => {
|
||||
expect(resolveEvalModel("capture", null, {} as never)).toBe("claude-opus-4-7");
|
||||
// capture defaults to Sonnet per D1a (2026-08 review): Opus is opt-in via
|
||||
// explicit arg or GSTACK_EVAL_MODEL_CAPTURE.
|
||||
expect(resolveEvalModel("capture", null, {} as never)).toBe("claude-sonnet-4-6");
|
||||
expect(resolveEvalModel("warmup", null, {} as never)).toBe("claude-haiku-4-5");
|
||||
expect(resolveEvalModel("distill", null, {} as never)).toBe("claude-haiku-4-5-20251001");
|
||||
});
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* Static tripwire for .github/workflows/free-tests.yml — the Linux free-suite
|
||||
* lane. Pins the three properties that made the lane worth having:
|
||||
*
|
||||
* 1. It invokes the CANONICAL runner (bun run test:free), not a raw
|
||||
* `bun test <dirs>` glob — the runner owns TEST_ROOTS and strict-output
|
||||
* classification, so a truncated run can't report green.
|
||||
* 2. It is SECRETLESS: free tests make no API calls, and keeping keys out
|
||||
* means fork PRs get real signal here. Any `secrets.` reference is a
|
||||
* regression.
|
||||
* 3. It triggers on `pull_request` (never `pull_request_target`, which
|
||||
* would hand a fork PR the base repo's context).
|
||||
*
|
||||
* Same wiring-tripwire class as test/hermetic-wiring.test.ts.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
const WORKFLOW = path.resolve(import.meta.dir, '..', '.github', 'workflows', 'free-tests.yml');
|
||||
|
||||
describe('free-tests workflow wiring', () => {
|
||||
const source = fs.readFileSync(WORKFLOW, 'utf-8');
|
||||
|
||||
test('workflow exists and invokes the canonical runner', () => {
|
||||
expect(source).toContain('bun run test:free');
|
||||
expect(source).not.toMatch(/run:\s*bun test\s/);
|
||||
});
|
||||
|
||||
test('secretless: no secrets reach the free lane', () => {
|
||||
expect(source).not.toContain('secrets.');
|
||||
expect(source).not.toContain('ANTHROPIC_API_KEY');
|
||||
expect(source).not.toContain('OPENAI_API_KEY');
|
||||
});
|
||||
|
||||
test('pull_request trigger, never pull_request_target', () => {
|
||||
expect(source).toContain('pull_request:');
|
||||
expect(source).not.toContain('pull_request_target');
|
||||
});
|
||||
|
||||
test('if sharded (matrix), the matrix count matches --shards N', () => {
|
||||
// Single-job --parallel mode has no matrix — vacuously fine. If someone
|
||||
// switches to the shard matrix (the V3 fallback), the two encodings of
|
||||
// the shard count must agree or CI silently drops files.
|
||||
const shardsFlag = source.match(/--shards\s+(\d+)/);
|
||||
const matrix = source.match(/shard:\s*\[([^\]]+)\]/);
|
||||
if (shardsFlag || matrix) {
|
||||
expect(shardsFlag, 'matrix present but no --shards N flag').toBeTruthy();
|
||||
expect(matrix, '--shards N present but no shard matrix').toBeTruthy();
|
||||
const count = parseInt(shardsFlag![1], 10);
|
||||
const entries = matrix![1].split(',').map(s => s.trim()).filter(Boolean);
|
||||
expect(entries.length).toBe(count);
|
||||
}
|
||||
});
|
||||
|
||||
test('least-privilege token: contents read-only, credentials not persisted', () => {
|
||||
// The job executes PR-controlled code (install lifecycle scripts + the
|
||||
// suite itself). A default-grant GITHUB_TOKEN persisted into .git/config
|
||||
// by checkout would hand that code whatever the repo default allows.
|
||||
expect(source).toMatch(/permissions:\s*\n\s*contents:\s*read/);
|
||||
expect(source).toMatch(/persist-credentials:\s*false/);
|
||||
});
|
||||
});
|
||||
@@ -25,7 +25,16 @@ const INSTALL = path.join(ROOT, 'bin', 'gstack-gbrain-install');
|
||||
// dirs — this keeps `gbrain` out of PATH deterministically across dev machines
|
||||
// while still finding jq, git, curl, sed, cat, etc. Each test can prepend a
|
||||
// fake-gbrain dir when it wants to simulate presence.
|
||||
const SAFE_PATH = '/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin:/usr/local/bin';
|
||||
// Deterministic PATH for spawned children — but it must still contain the
|
||||
// bun runtime itself: the bin's `#!/usr/bin/env -S bun run` shebang resolves
|
||||
// bun from PATH, and CI installs bun outside the standard dirs (~/.bun/bin),
|
||||
// which made every spawn exit 127 on the first Linux run. Appending bun's
|
||||
// REAL dir would leak its siblings (a dev box keeps gbrain in ~/.bun/bin
|
||||
// too, breaking every "no gbrain on PATH" case) — so a scratch dir holds a
|
||||
// symlink to bun and nothing else.
|
||||
const BUN_ONLY_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'bun-only-'));
|
||||
fs.symlinkSync(process.execPath, path.join(BUN_ONLY_DIR, 'bun'));
|
||||
const SAFE_PATH = `/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin:/usr/local/bin:${BUN_ONLY_DIR}`;
|
||||
|
||||
let tmpHome: string;
|
||||
let tmpHomeReal: string;
|
||||
|
||||
@@ -11,17 +11,28 @@
|
||||
* GET /config/database/pooler), PAT + DB_PASS env-var discipline, retry
|
||||
* + backoff on transient errors, pooler URL construction using the
|
||||
* generated DB_PASS (not the API response's templated connection_string).
|
||||
*
|
||||
* Tests drive lib/gbrain-supabase-provision.ts IN-PROCESS with injected
|
||||
* fetch/env/sleep (decision D7: dependency injection via options, never
|
||||
* process.env mutation before import — ESM hoists imports so env-set-before-
|
||||
* import silently doesn't work). One spawn-based smoke test at the bottom
|
||||
* runs the real bin end-to-end to pin the shebang/CLI contract. This
|
||||
* replaced ~30 process spawns (~16s of Bun boot + transpile) with direct
|
||||
* module calls.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, afterEach } from 'bun:test';
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
|
||||
import { runProvision } from '../lib/gbrain-supabase-provision';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const BIN = path.join(ROOT, 'bin', 'gstack-gbrain-supabase-provision');
|
||||
|
||||
// Minimal PATH that finds jq/curl but excludes user bins.
|
||||
// Minimal PATH that finds standard tools but excludes user bins. The smoke
|
||||
// test prepends the running bun's own directory so the shebang resolves.
|
||||
const SAFE_PATH = '/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin:/usr/local/bin';
|
||||
|
||||
type Handler = (req: Request) => Response | Promise<Response>;
|
||||
@@ -60,23 +71,29 @@ function startMock(routes: Record<string, Handler>): MockServer {
|
||||
};
|
||||
}
|
||||
|
||||
async function runBin(
|
||||
// Per-test GSTACK_HOME so egress receipts land in a throwaway ledger, never
|
||||
// the operator's real ~/.gstack/security/egress.jsonl.
|
||||
let egressHome: string;
|
||||
|
||||
/**
|
||||
* Run the CLI in-process with injected fetch (real fetch — it round-trips to
|
||||
* the Bun.serve loopback mock), injected env (the module never reads
|
||||
* process.env), and a no-op sleep so retry/backoff and wait-poll paths run
|
||||
* instantly.
|
||||
*/
|
||||
async function runCmd(
|
||||
args: string[],
|
||||
env: Record<string, string> = {}
|
||||
): Promise<{ stdout: string; stderr: string; status: number }> {
|
||||
// Use Bun.spawn (async) rather than spawnSync. spawnSync blocks the Bun
|
||||
// event loop, which prevents Bun.serve mocks from responding — every
|
||||
// HTTP call would hit curl's timeout instead of round-tripping.
|
||||
const proc = Bun.spawn([BIN, ...args], {
|
||||
env: { PATH: SAFE_PATH, ...env },
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
const status = await runProvision(args, {
|
||||
fetch: globalThis.fetch,
|
||||
env: { GSTACK_HOME: egressHome, ...env },
|
||||
stdout: (chunk) => { stdout += chunk; },
|
||||
stderr: (chunk) => { stderr += chunk; },
|
||||
sleep: async () => {},
|
||||
});
|
||||
const [stdout, stderr, status] = await Promise.all([
|
||||
new Response(proc.stdout).text(),
|
||||
new Response(proc.stderr).text(),
|
||||
proc.exited,
|
||||
]);
|
||||
return { stdout: stdout.trim(), stderr: stderr.trim(), status };
|
||||
}
|
||||
|
||||
@@ -89,8 +106,13 @@ function jsonResp(body: any, status = 200): Response {
|
||||
|
||||
let mock: MockServer;
|
||||
|
||||
beforeEach(() => {
|
||||
egressHome = fs.mkdtempSync(path.join(os.tmpdir(), 'provision-egress-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (mock) mock.close();
|
||||
fs.rmSync(egressHome, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('list-orgs', () => {
|
||||
@@ -102,7 +124,7 @@ describe('list-orgs', () => {
|
||||
{ id: 'deprec-2', slug: 'personal', name: 'Personal' },
|
||||
]),
|
||||
});
|
||||
const r = await runBin(['list-orgs', '--json'], {
|
||||
const r = await runCmd(['list-orgs', '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test_pat',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
});
|
||||
@@ -122,7 +144,7 @@ describe('list-orgs', () => {
|
||||
return jsonResp([]);
|
||||
},
|
||||
});
|
||||
await runBin(['list-orgs', '--json'], {
|
||||
await runCmd(['list-orgs', '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_expected_pat_xxx',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
});
|
||||
@@ -130,7 +152,7 @@ describe('list-orgs', () => {
|
||||
});
|
||||
|
||||
test('exits 3 with auth error when SUPABASE_ACCESS_TOKEN is missing', async () => {
|
||||
const r = await runBin(['list-orgs']);
|
||||
const r = await runCmd(['list-orgs']);
|
||||
expect(r.status).toBe(3);
|
||||
expect(r.stderr).toContain('SUPABASE_ACCESS_TOKEN is not set');
|
||||
});
|
||||
@@ -139,7 +161,7 @@ describe('list-orgs', () => {
|
||||
mock = startMock({
|
||||
'GET /v1/organizations': () => jsonResp({ message: 'Invalid JWT' }, 401),
|
||||
});
|
||||
const r = await runBin(['list-orgs'], {
|
||||
const r = await runCmd(['list-orgs'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_bad',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
});
|
||||
@@ -151,7 +173,7 @@ describe('list-orgs', () => {
|
||||
mock = startMock({
|
||||
'GET /v1/organizations': () => jsonResp({ message: 'Forbidden' }, 403),
|
||||
});
|
||||
const r = await runBin(['list-orgs'], {
|
||||
const r = await runCmd(['list-orgs'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_noperm',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
});
|
||||
@@ -177,7 +199,7 @@ describe('create', () => {
|
||||
}, 201);
|
||||
},
|
||||
});
|
||||
const r = await runBin(['create', 'gbrain', 'us-east-1', 'acme', '--json'], {
|
||||
const r = await runCmd(['create', 'gbrain', 'us-east-1', 'acme', '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
DB_PASS: 'generated-secret-pw',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
@@ -203,7 +225,7 @@ describe('create', () => {
|
||||
return jsonResp({ ref: 'r', status: 'COMING_UP' }, 201);
|
||||
},
|
||||
});
|
||||
await runBin(['create', 'gbrain', 'us-east-1', 'acme', '--instance-size', 'small', '--json'], {
|
||||
await runCmd(['create', 'gbrain', 'us-east-1', 'acme', '--instance-size', 'small', '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
DB_PASS: 'pw',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
@@ -215,7 +237,7 @@ describe('create', () => {
|
||||
mock = startMock({
|
||||
'POST /v1/projects': () => jsonResp({ message: 'project limit reached' }, 402),
|
||||
});
|
||||
const r = await runBin(['create', 'gbrain', 'us-east-1', 'acme'], {
|
||||
const r = await runCmd(['create', 'gbrain', 'us-east-1', 'acme'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
DB_PASS: 'pw',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
@@ -229,7 +251,7 @@ describe('create', () => {
|
||||
mock = startMock({
|
||||
'POST /v1/projects': () => jsonResp({ message: 'conflict' }, 409),
|
||||
});
|
||||
const r = await runBin(['create', 'gbrain', 'us-east-1', 'acme'], {
|
||||
const r = await runCmd(['create', 'gbrain', 'us-east-1', 'acme'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
DB_PASS: 'pw',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
@@ -240,7 +262,7 @@ describe('create', () => {
|
||||
});
|
||||
|
||||
test('fails when DB_PASS is missing', async () => {
|
||||
const r = await runBin(['create', 'gbrain', 'us-east-1', 'acme'], {
|
||||
const r = await runCmd(['create', 'gbrain', 'us-east-1', 'acme'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
});
|
||||
expect(r.status).toBe(2);
|
||||
@@ -248,7 +270,7 @@ describe('create', () => {
|
||||
});
|
||||
|
||||
test('missing positional args rejected with exit 2', async () => {
|
||||
const r = await runBin(['create', 'gbrain'], {
|
||||
const r = await runCmd(['create', 'gbrain'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
DB_PASS: 'pw',
|
||||
});
|
||||
@@ -265,14 +287,14 @@ describe('create', () => {
|
||||
return jsonResp({ ref: 'r', status: 'COMING_UP' }, 201);
|
||||
},
|
||||
});
|
||||
const r = await runBin(['create', 'gbrain', 'us-east-1', 'acme', '--json'], {
|
||||
const r = await runCmd(['create', 'gbrain', 'us-east-1', 'acme', '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
DB_PASS: 'pw',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
});
|
||||
expect(r.status).toBe(0);
|
||||
expect(count).toBe(2);
|
||||
}, 15000);
|
||||
});
|
||||
|
||||
test('exits 8 on persistent 5xx after max retries', async () => {
|
||||
let count = 0;
|
||||
@@ -282,7 +304,7 @@ describe('create', () => {
|
||||
return jsonResp({ message: 'internal server error' }, 502);
|
||||
},
|
||||
});
|
||||
const r = await runBin(['create', 'gbrain', 'us-east-1', 'acme'], {
|
||||
const r = await runCmd(['create', 'gbrain', 'us-east-1', 'acme'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
DB_PASS: 'pw',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
@@ -290,7 +312,7 @@ describe('create', () => {
|
||||
expect(r.status).toBe(8);
|
||||
expect(r.stderr).toContain('502');
|
||||
expect(count).toBeGreaterThanOrEqual(3);
|
||||
}, 30000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('wait', () => {
|
||||
@@ -303,7 +325,7 @@ describe('wait', () => {
|
||||
return jsonResp({ ref: 'abc', status: 'ACTIVE_HEALTHY' });
|
||||
},
|
||||
});
|
||||
const r = await runBin(['wait', 'abc', '--timeout', '30', '--json'], {
|
||||
const r = await runCmd(['wait', 'abc', '--timeout', '30', '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
});
|
||||
@@ -311,13 +333,13 @@ describe('wait', () => {
|
||||
const j = JSON.parse(r.stdout);
|
||||
expect(j.status).toBe('ACTIVE_HEALTHY');
|
||||
expect(j.ref).toBe('abc');
|
||||
}, 30000);
|
||||
});
|
||||
|
||||
test('exits 7 on terminal INIT_FAILED state', async () => {
|
||||
mock = startMock({
|
||||
'GET /v1/projects/abc': () => jsonResp({ ref: 'abc', status: 'INIT_FAILED' }),
|
||||
});
|
||||
const r = await runBin(['wait', 'abc', '--timeout', '10'], {
|
||||
const r = await runCmd(['wait', 'abc', '--timeout', '10'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
});
|
||||
@@ -330,14 +352,25 @@ describe('wait', () => {
|
||||
mock = startMock({
|
||||
'GET /v1/projects/abc': () => jsonResp({ ref: 'abc', status: 'COMING_UP' }),
|
||||
});
|
||||
const r = await runBin(['wait', 'abc', '--timeout', '0'], {
|
||||
const r = await runCmd(['wait', 'abc', '--timeout', '0'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
});
|
||||
expect(r.status).toBe(6);
|
||||
expect(r.stderr).toContain('wait timed out');
|
||||
expect(r.stderr).toContain('--resume-provision abc');
|
||||
}, 15000);
|
||||
});
|
||||
|
||||
test('non-numeric --timeout dies at parse time instead of polling forever', async () => {
|
||||
// NaN would make `elapsed >= timeout` always false: an infinite 5s poll loop.
|
||||
// The bash predecessor errored on `[ "$elapsed" -ge "abc" ]`; the port
|
||||
// must be at least as strict.
|
||||
const r = await runCmd(['wait', 'abc', '--timeout', 'abc'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
});
|
||||
expect(r.status).toBe(2);
|
||||
expect(r.stderr).toContain('--timeout must be a non-negative integer');
|
||||
});
|
||||
});
|
||||
|
||||
describe('pooler-url', () => {
|
||||
@@ -356,7 +389,7 @@ describe('pooler-url', () => {
|
||||
mock = startMock({
|
||||
[`GET /v1/projects/${REF}/config/database/pooler`]: () => jsonResp(POOLER_OK),
|
||||
});
|
||||
const r = await runBin(['pooler-url', REF, '--json'], {
|
||||
const r = await runCmd(['pooler-url', REF, '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
DB_PASS: 'my-real-password',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
@@ -370,6 +403,29 @@ describe('pooler-url', () => {
|
||||
expect(j.pooler_url).not.toContain('[PASSWORD]');
|
||||
});
|
||||
|
||||
test('percent-encodes reserved characters in DB_PASS (DSN stays parseable)', async () => {
|
||||
// Raw interpolation of a password containing / # ? % @ changes URI
|
||||
// structure: provisioning succeeds, every consumer then fails to parse
|
||||
// the DSN — an unusable billable orphan.
|
||||
mock = startMock({
|
||||
[`GET /v1/projects/${REF}/config/database/pooler`]: () => jsonResp(POOLER_OK),
|
||||
});
|
||||
const r = await runCmd(['pooler-url', REF, '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
DB_PASS: 'p@ss/w#rd?100%',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
});
|
||||
expect(r.status).toBe(0);
|
||||
const j = JSON.parse(r.stdout);
|
||||
// Expected URL assembled from parts so this file's own pushed bytes never
|
||||
// form a contiguous scheme://user:pass@host credential shape.
|
||||
const expectedUrl = 'postgresql://postgres.' + REF + ':' + encodeURIComponent('p@ss/w#rd?100%')
|
||||
+ '@' + 'aws-0-us-east-1.pooler.supabase.com:6543/postgres';
|
||||
expect(j.pooler_url).toBe(expectedUrl);
|
||||
// The password segment must parse back out intact.
|
||||
expect(decodeURIComponent(new URL(j.pooler_url).password)).toBe('p@ss/w#rd?100%');
|
||||
});
|
||||
|
||||
test('handles array response by preferring session pool_mode entry', async () => {
|
||||
mock = startMock({
|
||||
[`GET /v1/projects/${REF}/config/database/pooler`]: () =>
|
||||
@@ -378,7 +434,7 @@ describe('pooler-url', () => {
|
||||
{ ...POOLER_OK, pool_mode: 'session', db_port: 5432 },
|
||||
]),
|
||||
});
|
||||
const r = await runBin(['pooler-url', REF, '--json'], {
|
||||
const r = await runCmd(['pooler-url', REF, '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
DB_PASS: 'pw',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
@@ -394,7 +450,7 @@ describe('pooler-url', () => {
|
||||
[`GET /v1/projects/${REF}/config/database/pooler`]: () =>
|
||||
jsonResp({ identifier: 'x', pool_mode: 'session' }),
|
||||
});
|
||||
const r = await runBin(['pooler-url', REF], {
|
||||
const r = await runCmd(['pooler-url', REF], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
DB_PASS: 'pw',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
@@ -404,7 +460,7 @@ describe('pooler-url', () => {
|
||||
});
|
||||
|
||||
test('requires DB_PASS to construct URL', async () => {
|
||||
const r = await runBin(['pooler-url', REF], {
|
||||
const r = await runCmd(['pooler-url', REF], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
});
|
||||
expect(r.status).toBe(2);
|
||||
@@ -420,7 +476,7 @@ describe('pooler-url', () => {
|
||||
[`GET /v1/projects/${REF}/config/database/pooler`]: () =>
|
||||
jsonResp({ ...POOLER_OK, pool_mode: 'transaction', db_port: 6543 }),
|
||||
});
|
||||
const r = await runBin(['pooler-url', REF, '--json'], {
|
||||
const r = await runCmd(['pooler-url', REF, '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
DB_PASS: 'pw',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
@@ -435,7 +491,7 @@ describe('pooler-url', () => {
|
||||
[`GET /v1/projects/${REF}/config/database/pooler`]: () =>
|
||||
jsonResp({ ...POOLER_OK, pool_mode: 'session', db_port: 6543 }),
|
||||
});
|
||||
const r = await runBin(['pooler-url', REF, '--json'], {
|
||||
const r = await runCmd(['pooler-url', REF, '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
DB_PASS: 'pw',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
@@ -450,7 +506,7 @@ describe('pooler-url', () => {
|
||||
[`GET /v1/projects/${REF}/config/database/pooler`]: () =>
|
||||
jsonResp({ ...POOLER_OK, pool_mode: 'transaction', db_port: 5432 }),
|
||||
});
|
||||
const r = await runBin(['pooler-url', REF, '--json'], {
|
||||
const r = await runCmd(['pooler-url', REF, '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
DB_PASS: 'pw',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
@@ -465,7 +521,7 @@ describe('pooler-url', () => {
|
||||
[`GET /v1/projects/${REF}/config/database/pooler`]: () =>
|
||||
jsonResp({ ...POOLER_OK, pool_mode: 'transaction', db_port: 6543 }),
|
||||
});
|
||||
const r = await runBin(['pooler-url', REF, '--json'], {
|
||||
const r = await runCmd(['pooler-url', REF, '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
DB_PASS: 'pw',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
@@ -484,7 +540,7 @@ describe('pooler-url', () => {
|
||||
{ ...POOLER_OK, pool_mode: 'session', db_port: 5432 },
|
||||
]),
|
||||
});
|
||||
const r = await runBin(['pooler-url', REF, '--json'], {
|
||||
const r = await runCmd(['pooler-url', REF, '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
DB_PASS: 'pw',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
@@ -519,7 +575,7 @@ describe('list-orphans (D20)', () => {
|
||||
})
|
||||
);
|
||||
try {
|
||||
const r = await runBin(['list-orphans', '--json'], {
|
||||
const r = await runCmd(['list-orphans', '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
HOME: home,
|
||||
@@ -543,7 +599,7 @@ describe('list-orphans (D20)', () => {
|
||||
});
|
||||
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gbrain-no-cfg-'));
|
||||
try {
|
||||
const r = await runBin(['list-orphans', '--json'], {
|
||||
const r = await runCmd(['list-orphans', '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
HOME: home,
|
||||
@@ -569,7 +625,7 @@ describe('list-orphans (D20)', () => {
|
||||
});
|
||||
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gbrain-prefix-'));
|
||||
try {
|
||||
const r = await runBin(['list-orphans', '--name-prefix', 'my-prefix', '--json'], {
|
||||
const r = await runCmd(['list-orphans', '--name-prefix', 'my-prefix', '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
HOME: home,
|
||||
@@ -593,7 +649,7 @@ describe('delete-project (D20)', () => {
|
||||
return jsonResp({ id: 1, ref: 'abcdefghijklmnopqrst', name: 'gbrain' });
|
||||
},
|
||||
});
|
||||
const r = await runBin(['delete-project', 'abcdefghijklmnopqrst', '--json'], {
|
||||
const r = await runCmd(['delete-project', 'abcdefghijklmnopqrst', '--json'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
});
|
||||
@@ -607,7 +663,7 @@ describe('delete-project (D20)', () => {
|
||||
mock = startMock({
|
||||
'DELETE /v1/projects/nonexistent': () => jsonResp({ message: 'Project not found' }, 404),
|
||||
});
|
||||
const r = await runBin(['delete-project', 'nonexistent'], {
|
||||
const r = await runCmd(['delete-project', 'nonexistent'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
});
|
||||
@@ -616,7 +672,7 @@ describe('delete-project (D20)', () => {
|
||||
});
|
||||
|
||||
test('requires a ref', async () => {
|
||||
const r = await runBin(['delete-project'], {
|
||||
const r = await runCmd(['delete-project'], {
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_test',
|
||||
});
|
||||
expect(r.status).toBe(2);
|
||||
@@ -626,14 +682,65 @@ describe('delete-project (D20)', () => {
|
||||
|
||||
describe('general', () => {
|
||||
test('unknown subcommand exits 2', async () => {
|
||||
const r = await runBin(['nope']);
|
||||
const r = await runCmd(['nope']);
|
||||
expect(r.status).toBe(2);
|
||||
expect(r.stderr).toContain('unknown subcommand');
|
||||
});
|
||||
|
||||
test('no args prints usage and exits 2', async () => {
|
||||
const r = await runBin([]);
|
||||
const r = await runCmd([]);
|
||||
expect(r.status).toBe(2);
|
||||
expect(r.stderr).toContain('usage');
|
||||
});
|
||||
|
||||
test('--help prints the doc header and exits 0', async () => {
|
||||
const r = await runCmd(['--help']);
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout).toContain(
|
||||
'gstack-gbrain-supabase-provision — Supabase Management API wrapper'
|
||||
);
|
||||
expect(r.stdout).toContain('Exit codes:');
|
||||
});
|
||||
});
|
||||
|
||||
describe('bin smoke test (spawned)', () => {
|
||||
// Exactly one spawn-based test: runs the real bin end-to-end against the
|
||||
// mock server to pin the shebang/CLI contract (bun-shebang resolves, argv
|
||||
// and env flow through, JSON lands on stdout, exit code propagates). All
|
||||
// behavioral coverage above runs in-process.
|
||||
test('real bin: list-orgs --json round-trips against a mock server', async () => {
|
||||
let authHeader = '';
|
||||
mock = startMock({
|
||||
'GET /v1/organizations': (req) => {
|
||||
authHeader = req.headers.get('authorization') || '';
|
||||
return jsonResp([{ id: 'x', slug: 'acme', name: 'Acme Inc' }]);
|
||||
},
|
||||
});
|
||||
// Use Bun.spawn (async) rather than spawnSync. spawnSync blocks the Bun
|
||||
// event loop, which prevents Bun.serve mocks from responding — every
|
||||
// HTTP call would hit fetch's timeout instead of round-tripping.
|
||||
const proc = Bun.spawn([BIN, 'list-orgs', '--json'], {
|
||||
env: {
|
||||
PATH: `${path.dirname(process.execPath)}:${SAFE_PATH}`,
|
||||
SUPABASE_ACCESS_TOKEN: 'sbp_smoke_pat',
|
||||
SUPABASE_API_BASE: mock.url,
|
||||
GSTACK_HOME: egressHome,
|
||||
},
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
});
|
||||
const [stdout, stderr, status] = await Promise.all([
|
||||
new Response(proc.stdout).text(),
|
||||
new Response(proc.stderr).text(),
|
||||
proc.exited,
|
||||
]);
|
||||
expect(status).toBe(0);
|
||||
expect(stderr.trim()).toBe('');
|
||||
expect(authHeader).toBe('Bearer sbp_smoke_pat');
|
||||
expect(JSON.parse(stdout.trim())).toEqual({ orgs: [{ slug: 'acme', name: 'Acme Inc' }] });
|
||||
// The spawned bin wrote its egress receipt into the per-test ledger.
|
||||
const ledger = path.join(egressHome, 'security', 'egress.jsonl');
|
||||
expect(fs.existsSync(ledger)).toBe(true);
|
||||
expect(fs.readFileSync(ledger, 'utf-8')).toContain('"sink":"supabase-provision"');
|
||||
}, 20_000);
|
||||
});
|
||||
|
||||
@@ -299,7 +299,10 @@ export async function runAgentSdkTest(
|
||||
const sem = getApiSemaphore();
|
||||
const maxRetries = opts.maxRetries ?? 3;
|
||||
const queryImpl: QueryProvider = opts.queryProvider ?? query;
|
||||
const model = opts.model ?? 'claude-opus-4-7';
|
||||
// Default matches session-runner's Sonnet (D1a, 2026-08): the old Opus
|
||||
// default was an inconsistency between the two runners, not a choice —
|
||||
// tests that need Opus pin it via opts.model (30+ already do).
|
||||
const model = opts.model ?? 'claude-sonnet-4-6';
|
||||
|
||||
// NOTE on env: the SDK child gets the COMPLETE hermetic env (allowlist
|
||||
// scrub + ANTHROPIC_API_KEY + hermetic CLAUDE_CONFIG_DIR/GSTACK_HOME), with
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
/**
|
||||
* Anthropic-API preflight ping, shared by e2e-helpers (module load in every
|
||||
* paid test file) and the sharded paid runner (once, in the parent).
|
||||
*
|
||||
* The ping is a real `claude -p` call with a 30s timeout. Before the parent
|
||||
* dedup, every one of the ~30 paid test files that import e2e-helpers fired
|
||||
* it at module load — 30 paid pings per full sharded run for one bit of
|
||||
* information. The sharded runner now pings ONCE and sets
|
||||
* EVALS_PREFLIGHT_OK=1 in each shard's env; the module-load path honors the
|
||||
* flag and skips.
|
||||
*
|
||||
* Lives in its own module (not e2e-helpers) so the runner can import it
|
||||
* without dragging in bun:test.
|
||||
*/
|
||||
|
||||
import { spawnSync } from 'child_process';
|
||||
|
||||
export type PreflightResult = 'skipped' | 'ok';
|
||||
|
||||
export function preflightAnthropicApi(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
spawn: typeof spawnSync = spawnSync,
|
||||
): PreflightResult {
|
||||
if (env.EVALS_PREFLIGHT_OK === '1') return 'skipped';
|
||||
const check = spawn(
|
||||
'sh',
|
||||
['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'],
|
||||
{ stdio: 'pipe', timeout: 30_000 },
|
||||
);
|
||||
// Fail fast on the failure modes that guarantee EVERY shard fails too:
|
||||
// spawn error (sh/claude unlaunchable), a timeout kill (signal set), or
|
||||
// exit 127 (command not found). Anything else stays fail-open — a flaky
|
||||
// preflight must not block a run the shards could complete (auth prompts
|
||||
// and transient non-zero exits are the shards' problem to report).
|
||||
if (check.error) {
|
||||
throw new Error(`Anthropic preflight could not run (${check.error.message}) — aborting E2E suite before spawning shards.`);
|
||||
}
|
||||
if (check.signal) {
|
||||
throw new Error(`Anthropic preflight timed out (killed with ${check.signal}) — aborting E2E suite. Check connectivity/auth and retry.`);
|
||||
}
|
||||
if (check.status === 127) {
|
||||
throw new Error('Anthropic preflight: `claude` not found on PATH (exit 127) — aborting E2E suite before spawning shards.');
|
||||
}
|
||||
const output = check.stdout?.toString() || '';
|
||||
if (output.includes('ConnectionRefused') || output.includes('Unable to connect')) {
|
||||
throw new Error('Anthropic API unreachable — aborting E2E suite. Fix connectivity and retry.');
|
||||
}
|
||||
return 'ok';
|
||||
}
|
||||
@@ -54,10 +54,18 @@ export interface ParityBaseline {
|
||||
export interface CaptureOptions {
|
||||
repoRoot: string;
|
||||
tag?: string;
|
||||
/**
|
||||
* Skills whose baseline bytes must be the UNION of skeleton + sections/*.md
|
||||
* (mirroring parity-harness readSkillForParity, which is what the checker
|
||||
* compares against). Omitting a carved skill here records skeleton-only
|
||||
* bytes and the ratio check then reads ~2x on the next parity run — the
|
||||
* exact capture-vs-check drift that broke the v1.64 rebase.
|
||||
*/
|
||||
sectionedSkills?: string[];
|
||||
}
|
||||
|
||||
/** Extract the frontmatter description from a SKILL.md file. Empty string if none. */
|
||||
function extractDescription(content: string): string {
|
||||
export function extractDescription(content: string): string {
|
||||
if (!content.startsWith('---\n')) return '';
|
||||
const fmEnd = content.indexOf('\n---', 4);
|
||||
if (fmEnd === -1) return '';
|
||||
@@ -142,7 +150,7 @@ function getGitInfo(repoRoot: string): { commit: string; branch: string } {
|
||||
}
|
||||
|
||||
export function captureBaseline(opts: CaptureOptions): ParityBaseline {
|
||||
const { repoRoot, tag } = opts;
|
||||
const { repoRoot, tag, sectionedSkills } = opts;
|
||||
const skillDirs = discoverSkillDirs(repoRoot);
|
||||
const evalCoverage = discoverEvalCoverage(repoRoot, skillDirs);
|
||||
const skills: Record<string, SkillBaselineEntry> = {};
|
||||
@@ -152,7 +160,18 @@ export function captureBaseline(opts: CaptureOptions): ParityBaseline {
|
||||
const skillMdPath = path.join(repoRoot, dir, 'SKILL.md');
|
||||
const tmplPath = path.join(repoRoot, dir, 'SKILL.md.tmpl');
|
||||
const content = fs.readFileSync(skillMdPath, 'utf-8');
|
||||
const bytes = Buffer.byteLength(content, 'utf-8');
|
||||
let bytes = Buffer.byteLength(content, 'utf-8');
|
||||
// Union in the carved sections for sectioned skills — semantic twin of
|
||||
// parity-harness readSkillForParity (which the checker uses). Kept inline
|
||||
// because parity-harness imports this module as a value (cycle).
|
||||
if (sectionedSkills?.includes(dir)) {
|
||||
const sectionsDir = path.join(repoRoot, dir, 'sections');
|
||||
if (fs.existsSync(sectionsDir)) {
|
||||
for (const f of fs.readdirSync(sectionsDir).filter(f => f.endsWith('.md')).sort()) {
|
||||
bytes += Buffer.byteLength(fs.readFileSync(path.join(sectionsDir, f), 'utf-8'), 'utf-8');
|
||||
}
|
||||
}
|
||||
}
|
||||
const lines = content.split('\n').length;
|
||||
const description = extractDescription(content);
|
||||
const descriptionLen = Buffer.byteLength(description, 'utf-8');
|
||||
|
||||
@@ -16,6 +16,7 @@ import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { hermeticChildEnv } from './hermetic-env';
|
||||
import { extractSkillSections } from './skill-fixture';
|
||||
|
||||
// --- Interfaces ---
|
||||
|
||||
@@ -103,19 +104,32 @@ export function parseCodexJSONL(lines: string[]): ParsedCodexJSONL {
|
||||
* Creates ~/.codex/skills/{skillName}/SKILL.md in the temp HOME and copies
|
||||
* agents/openai.yaml when present so Codex sees the same metadata as a real install.
|
||||
*
|
||||
* When `sections` is provided, the installed SKILL.md is an EXTRACTION
|
||||
* (frontmatter + the named `## <section>` blocks via
|
||||
* test/helpers/skill-fixture.ts) instead of the full 1000-1900-line file —
|
||||
* CLAUDE.md: "E2E test fixtures: extract, don't copy". Omit `sections` only
|
||||
* when the test's purpose is to validate the real generated artifact itself
|
||||
* (e.g., codex-discover-skill asserts the full SKILL.md loads without
|
||||
* "invalid" / "Skipped loading" stderr from Codex).
|
||||
*
|
||||
* Returns the temp HOME path. Caller is responsible for cleanup.
|
||||
*/
|
||||
export function installSkillToTempHome(
|
||||
skillDir: string,
|
||||
skillName: string,
|
||||
tempHome?: string,
|
||||
sections?: string[],
|
||||
): string {
|
||||
const home = tempHome || fs.mkdtempSync(path.join(os.tmpdir(), 'codex-e2e-'));
|
||||
const destDir = path.join(home, '.codex', 'skills', skillName);
|
||||
fs.mkdirSync(destDir, { recursive: true });
|
||||
|
||||
const srcSkill = path.join(skillDir, 'SKILL.md');
|
||||
if (fs.existsSync(srcSkill)) {
|
||||
if (sections && sections.length > 0) {
|
||||
// extractSkillSections throws loudly on a missing file or renamed
|
||||
// section — a fixture is never silently written empty.
|
||||
fs.writeFileSync(path.join(destDir, 'SKILL.md'), extractSkillSections(skillDir, sections));
|
||||
} else if (fs.existsSync(srcSkill)) {
|
||||
fs.copyFileSync(srcSkill, path.join(destDir, 'SKILL.md'));
|
||||
}
|
||||
|
||||
@@ -144,6 +158,7 @@ export async function runCodexSkill(opts: {
|
||||
cwd?: string; // Working directory
|
||||
skillName?: string; // Skill name for installation (default: dirname)
|
||||
sandbox?: string; // Sandbox mode (default: 'read-only')
|
||||
sections?: string[]; // Install only these `## <section>` blocks (extract, don't copy)
|
||||
}): Promise<CodexResult> {
|
||||
const {
|
||||
skillDir,
|
||||
@@ -152,6 +167,7 @@ export async function runCodexSkill(opts: {
|
||||
cwd,
|
||||
skillName,
|
||||
sandbox = 'read-only',
|
||||
sections,
|
||||
} = opts;
|
||||
|
||||
const startTime = Date.now();
|
||||
@@ -178,7 +194,7 @@ export async function runCodexSkill(opts: {
|
||||
const realHome = os.homedir();
|
||||
|
||||
try {
|
||||
installSkillToTempHome(skillDir, name, tempHome);
|
||||
installSkillToTempHome(skillDir, name, tempHome, sections);
|
||||
|
||||
// Symlink real Codex auth config so codex can authenticate from temp HOME.
|
||||
// Codex stores auth in ~/.codex/ — we need the config but not the skills
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
/**
|
||||
* Shared helpers for E2E test files.
|
||||
*
|
||||
* Extracted from the monolithic skill-e2e.test.ts to support splitting
|
||||
* tests across multiple files by category.
|
||||
* Extracted from the (since-deleted) pre-split monolith to support
|
||||
* splitting tests across multiple skill-e2e-*.test.ts files by category.
|
||||
*/
|
||||
|
||||
import '../../lib/conductor-env-shim';
|
||||
@@ -15,6 +15,7 @@ import { selectTests, detectBaseBranch, getChangedFiles, E2E_TOUCHFILES, E2E_TIE
|
||||
import { WorktreeManager } from '../../lib/worktree';
|
||||
import type { HarvestResult } from '../../lib/worktree';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { preflightAnthropicApi } from './anthropic-preflight';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
@@ -275,15 +276,12 @@ if (evalsEnabled) {
|
||||
}
|
||||
}
|
||||
|
||||
// Fail fast if Anthropic API is unreachable — don't burn through tests getting ConnectionRefused
|
||||
// Fail fast if Anthropic API is unreachable — don't burn through tests getting
|
||||
// ConnectionRefused. The sharded paid runner pings once in the parent and sets
|
||||
// EVALS_PREFLIGHT_OK=1 for its children, so per-file module loads skip this
|
||||
// (was: ~30 paid pings per full sharded run, one per importing file).
|
||||
if (evalsEnabled) {
|
||||
const check = spawnSync('sh', ['-c', 'echo "ping" | claude -p --max-turns 1 --output-format stream-json --verbose --dangerously-skip-permissions'], {
|
||||
stdio: 'pipe', timeout: 30_000,
|
||||
});
|
||||
const output = check.stdout?.toString() || '';
|
||||
if (output.includes('ConnectionRefused') || output.includes('Unable to connect')) {
|
||||
throw new Error('Anthropic API unreachable — aborting E2E suite. Fix connectivity and retry.');
|
||||
}
|
||||
preflightAnthropicApi();
|
||||
}
|
||||
|
||||
/** Skip an individual test if not selected (for multi-test describe blocks). */
|
||||
|
||||
@@ -56,7 +56,17 @@ export interface RecommendationScore {
|
||||
* existing callers; pass a model id (e.g. claude-haiku-4-5-20251001)
|
||||
* for cheaper bounded judgments like judgeRecommendation.
|
||||
*/
|
||||
export async function callJudge<T>(prompt: string, model: string = 'claude-sonnet-4-6'): Promise<T> {
|
||||
// Default judge model: Sonnet. D1a tried Haiku 4.5 here and the first live
|
||||
// run regressed the doc-rubric family — a controlled A/B on the identical
|
||||
// health-rubric prompt scored 2/2/2 under Haiku vs 4/3/4 under Sonnet (both
|
||||
// with coherent reasoning; Haiku is simply a harsher grader on long-document
|
||||
// rubrics, and every >=4 threshold in skill-llm-eval was calibrated against
|
||||
// months of Sonnet baselines). Per D1a's pin-on-regressors protocol the
|
||||
// default stays Sonnet; recalibrating the 25 rubrics for Haiku is separately
|
||||
// scoped work. Override per run with GSTACK_EVAL_MODEL_JUDGE; Haiku remains
|
||||
// the right default for classifier-grade duties (pty hung/working, warmup,
|
||||
// distill — see lib/eval-model.ts).
|
||||
export async function callJudge<T>(prompt: string, model: string = process.env.GSTACK_EVAL_MODEL_JUDGE || 'claude-sonnet-4-6'): Promise<T> {
|
||||
const client = new Anthropic();
|
||||
|
||||
const makeRequest = () => client.messages.create({
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
/**
|
||||
* Skill fixture extraction — enforces the CLAUDE.md rule "E2E test fixtures:
|
||||
* extract, don't copy".
|
||||
*
|
||||
* Full SKILL.md files are 1000-1900 lines. When `claude -p` (or `codex exec`)
|
||||
* reads a file that large, context bloat causes timeouts, flaky turn limits,
|
||||
* and tests that take 5-10x longer than necessary. Every E2E fixture that
|
||||
* needs skill content should extract ONLY the sections the test actually
|
||||
* exercises, through one of the three helpers here:
|
||||
*
|
||||
* - extractSkillSections(skillDir, sections)
|
||||
* frontmatter + the named `## <section>` blocks, concatenated in the
|
||||
* order given. For tests that exercise specific workflow steps.
|
||||
* - extractSkillBody(skillDir)
|
||||
* frontmatter + intro + everything AFTER the shared generated preamble
|
||||
* ("## Preamble (run first)" .. end of "## Plan Status Footer").
|
||||
* For tests that exercise the skill's ENTIRE specific flow but never
|
||||
* touch the ~780-line shared preamble.
|
||||
* - extractSkillHead(skillDir, bodyLineCount)
|
||||
* frontmatter + the first N body lines. For ROUTING / discovery tests,
|
||||
* where the agent only reads the frontmatter (name + description) to
|
||||
* decide which skill to invoke.
|
||||
*
|
||||
* Failure polarity: every extraction failure (missing file, missing
|
||||
* frontmatter, renamed section) THROWS with the offending name — a fixture is
|
||||
* never silently written empty. test/skill-fixture.test.ts pins the exported
|
||||
* section lists against the real generated SKILL.md files, so a section
|
||||
* rename fails the FREE suite instead of a paid E2E run.
|
||||
*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
// ─── Section lists shared by E2E fixtures and the free pin test ────────────
|
||||
// Keep these verbatim against the H2 headings in the generated SKILL.md files.
|
||||
// If gen-skill-docs renames a heading, test/skill-fixture.test.ts fails free.
|
||||
|
||||
/** /review E2E (sql-injection, enum-completeness, design-lite): the core
|
||||
* review workflow without the shared preamble, Review Army, or Fix-First. */
|
||||
export const REVIEW_E2E_SECTIONS = [
|
||||
'When to invoke this skill',
|
||||
'Step 0: Detect platform and base branch',
|
||||
'Step 1: Check branch',
|
||||
'Step 2: Read the checklist',
|
||||
'Step 2.5: Check for Greptile review comments',
|
||||
'Step 3: Get the diff',
|
||||
'Step 4: Critical pass (core review)',
|
||||
'Confidence Calibration',
|
||||
'Important Rules',
|
||||
];
|
||||
|
||||
/** Review Army E2E: core workflow + Scope Drift / Plan Completion Audit
|
||||
* (delivery-audit test) + Step 4.5 specialist dispatch (quality score,
|
||||
* JSON findings schema, MULTI-SPECIALIST consensus, Red Team). */
|
||||
export const REVIEW_ARMY_E2E_SECTIONS = [
|
||||
'When to invoke this skill',
|
||||
'Step 0: Detect platform and base branch',
|
||||
'Step 1: Check branch',
|
||||
'Step 1.5: Scope Drift Detection',
|
||||
'Step 2: Read the checklist',
|
||||
'Step 2.5: Check for Greptile review comments',
|
||||
'Step 3: Get the diff',
|
||||
'Step 4: Critical pass (core review)',
|
||||
'Confidence Calibration',
|
||||
'Step 4.5: Review Army — Specialist Dispatch',
|
||||
'Important Rules',
|
||||
];
|
||||
|
||||
/** /retro E2E (retro, retro-base-branch): the repo-scoped retro flow
|
||||
* (Steps 0-14 live under Instructions/Prior Learnings/Capture Learnings)
|
||||
* + the narrative report template. Global mode and Compare mode are not
|
||||
* exercised by the E2E tests and are dropped. */
|
||||
export const RETRO_E2E_SECTIONS = [
|
||||
'When to invoke this skill',
|
||||
'Step 0: Detect platform and base branch',
|
||||
'User-invocable',
|
||||
'Arguments',
|
||||
'Instructions',
|
||||
'Prior Learnings',
|
||||
'Capture Learnings',
|
||||
'Engineering Retro: [date range]',
|
||||
'Tone',
|
||||
'Important Rules',
|
||||
];
|
||||
|
||||
/** codex-review-findings E2E against the Codex host variant
|
||||
* (.agents/skills/gstack-review/SKILL.md). Same core workflow as
|
||||
* REVIEW_E2E_SECTIONS, minus "When to invoke this skill" (the Codex host
|
||||
* adapter does not emit that section). */
|
||||
export const CODEX_REVIEW_E2E_SECTIONS = [
|
||||
'Step 0: Detect platform and base branch',
|
||||
'Step 1: Check branch',
|
||||
'Step 2: Read the checklist',
|
||||
'Step 3: Get the diff',
|
||||
'Step 4: Critical pass (core review)',
|
||||
'Confidence Calibration',
|
||||
'Important Rules',
|
||||
];
|
||||
|
||||
// ─── Parsing internals ──────────────────────────────────────────────────────
|
||||
|
||||
/** First/last H2 headings of the shared preamble block that gen-skill-docs
|
||||
* emits into every tier >= 2 skill. extractSkillBody drops this range. */
|
||||
const SHARED_PREAMBLE_FIRST = 'Preamble (run first)';
|
||||
const SHARED_PREAMBLE_LAST = 'Plan Status Footer';
|
||||
|
||||
interface H2Section {
|
||||
heading: string;
|
||||
/** index of the heading line within bodyLines */
|
||||
start: number;
|
||||
/** one past the last line of the section (start of next H2, or EOF) */
|
||||
end: number;
|
||||
}
|
||||
|
||||
/** Accept either a skill directory or a direct path to a .md file. */
|
||||
function resolveSkillMd(skillDirOrFile: string): string {
|
||||
const file = skillDirOrFile.endsWith('.md')
|
||||
? skillDirOrFile
|
||||
: path.join(skillDirOrFile, 'SKILL.md');
|
||||
if (!fs.existsSync(file)) {
|
||||
throw new Error(`skill-fixture: no SKILL.md at ${file}`);
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
function splitFrontmatter(raw: string, file: string): { frontmatter: string; bodyLines: string[] } {
|
||||
const lines = raw.split('\n');
|
||||
if ((lines[0] ?? '').trim() !== '---') {
|
||||
throw new Error(`skill-fixture: ${file} does not start with YAML frontmatter ('---')`);
|
||||
}
|
||||
let close = -1;
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
if (lines[i].trim() === '---') { close = i; break; }
|
||||
}
|
||||
if (close === -1) {
|
||||
throw new Error(`skill-fixture: ${file} frontmatter never closes ('---' missing)`);
|
||||
}
|
||||
return {
|
||||
frontmatter: lines.slice(0, close + 1).join('\n'),
|
||||
bodyLines: lines.slice(close + 1),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan body lines for H2 sections, fence-aware: `## `-prefixed lines inside
|
||||
* ``` / ~~~ code fences are template content (e.g. the PLAN COMPLETION AUDIT
|
||||
* output format, the /context-save checkpoint template), NOT section
|
||||
* boundaries. Fences close only on a matching char of >= opening length,
|
||||
* per CommonMark, so 4-backtick fences embedding 3-backtick blocks work.
|
||||
*/
|
||||
function scanH2Sections(bodyLines: string[]): H2Section[] {
|
||||
const sections: H2Section[] = [];
|
||||
let fence: { ch: string; len: number } | null = null;
|
||||
|
||||
for (let i = 0; i < bodyLines.length; i++) {
|
||||
const line = bodyLines[i];
|
||||
const m = line.match(/^ {0,3}(`{3,}|~{3,})(.*)$/);
|
||||
if (m) {
|
||||
const ch = m[1][0];
|
||||
const len = m[1].length;
|
||||
if (!fence) {
|
||||
fence = { ch, len };
|
||||
} else if (fence.ch === ch && len >= fence.len && m[2].trim() === '') {
|
||||
fence = null;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!fence && line.startsWith('## ')) {
|
||||
sections.push({ heading: line.slice(3).trim(), start: i, end: bodyLines.length });
|
||||
}
|
||||
}
|
||||
for (let s = 0; s < sections.length - 1; s++) {
|
||||
sections[s].end = sections[s + 1].start;
|
||||
}
|
||||
return sections;
|
||||
}
|
||||
|
||||
function loadSkill(skillDirOrFile: string): {
|
||||
file: string;
|
||||
frontmatter: string;
|
||||
bodyLines: string[];
|
||||
sections: H2Section[];
|
||||
} {
|
||||
const file = resolveSkillMd(skillDirOrFile);
|
||||
const raw = fs.readFileSync(file, 'utf-8');
|
||||
const { frontmatter, bodyLines } = splitFrontmatter(raw, file);
|
||||
return { file, frontmatter, bodyLines, sections: scanH2Sections(bodyLines) };
|
||||
}
|
||||
|
||||
function findSection(sections: H2Section[], name: string, file: string): H2Section {
|
||||
const hit = sections.find((s) => s.heading === name)
|
||||
?? sections.find((s) => s.heading.startsWith(name));
|
||||
if (!hit) {
|
||||
const available = sections.map((s) => ` ## ${s.heading}`).join('\n');
|
||||
throw new Error(
|
||||
`skill-fixture: section "## ${name}" not found in ${file}.\n`
|
||||
+ 'The section may have been renamed — update the fixture section list '
|
||||
+ '(see test/helpers/skill-fixture.ts).\n'
|
||||
+ `Available H2 sections:\n${available}`,
|
||||
);
|
||||
}
|
||||
return hit;
|
||||
}
|
||||
|
||||
// ─── Public API ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Read the real SKILL.md under `skillDir` (or a direct .md path), slice each
|
||||
* requested `## <section>` block, and return frontmatter + the sections
|
||||
* concatenated in the order given. Throws loudly on a missing section.
|
||||
*/
|
||||
export function extractSkillSections(skillDir: string, sections: string[]): string {
|
||||
const { file, frontmatter, bodyLines, sections: all } = loadSkill(skillDir);
|
||||
const parts: string[] = [frontmatter, ''];
|
||||
for (const name of sections) {
|
||||
const hit = findSection(all, name, file);
|
||||
parts.push(bodyLines.slice(hit.start, hit.end).join('\n').trimEnd(), '');
|
||||
}
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Frontmatter + intro (everything before "## Preamble (run first)") + the
|
||||
* full skill-specific body (everything after the "## Plan Status Footer"
|
||||
* section). Use when a test exercises the whole skill flow: this drops the
|
||||
* ~780-line shared generated preamble and nothing else.
|
||||
*/
|
||||
export function extractSkillBody(skillDir: string): string {
|
||||
const { file, frontmatter, bodyLines, sections: all } = loadSkill(skillDir);
|
||||
const first = findSection(all, SHARED_PREAMBLE_FIRST, file);
|
||||
const last = findSection(all, SHARED_PREAMBLE_LAST, file);
|
||||
const intro = bodyLines.slice(0, first.start).join('\n').trimEnd();
|
||||
const tail = bodyLines.slice(last.end).join('\n').trimEnd();
|
||||
if (!tail) {
|
||||
throw new Error(
|
||||
`skill-fixture: ${file} has no content after "## ${SHARED_PREAMBLE_LAST}" — `
|
||||
+ 'refusing to write a preamble-only fixture.',
|
||||
);
|
||||
}
|
||||
return [frontmatter, '', intro, '', tail, ''].join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Frontmatter + the first `bodyLineCount` body lines. For routing/discovery
|
||||
* fixtures: skill selection reads the frontmatter name + description, so the
|
||||
* body is intentionally truncated.
|
||||
*/
|
||||
export function extractSkillHead(skillDir: string, bodyLineCount = 30): string {
|
||||
const { frontmatter, bodyLines } = loadSkill(skillDir);
|
||||
const head = bodyLines.slice(0, bodyLineCount).join('\n').trimEnd();
|
||||
return `${frontmatter}\n${head}\n\n<!-- body truncated by test/helpers/skill-fixture.ts — routing fixture needs frontmatter only -->\n`;
|
||||
}
|
||||
@@ -0,0 +1,403 @@
|
||||
/**
|
||||
* Diff-based test selection for E2E and LLM-judge evals — the LOGIC half.
|
||||
*
|
||||
* Each test declares which source files it depends on ("touchfiles") in
|
||||
* ./touchfiles-data.ts (literals only — see the note there). The test runner
|
||||
* computes changed files as the union of committed diff, staged + unstaged
|
||||
* diff, and untracked files — uncommitted work selects tests too — and only
|
||||
* runs tests whose dependencies were modified. Override with EVALS_ALL=1 to
|
||||
* run everything.
|
||||
*
|
||||
* When touchfiles-data.ts itself changed, selection uses MAP-DIFF instead of
|
||||
* a global run-all: the old version of the data file is loaded from git and
|
||||
* evaluated in a bun child process (the literal-only tripwire in
|
||||
* test/touchfiles-facade.test.ts bounds what executing it can do), the four
|
||||
* maps are diffed per key, and only tests whose entry was added, whose
|
||||
* dep-list changed, or whose tier flipped are selected. Any failure on that
|
||||
* path fails CLOSED: run all tests, with the cause in the reason string.
|
||||
*
|
||||
* Everything here is synchronous by design: e2e-helpers.ts and the *-e2e
|
||||
* test files compute selection at module load, so the old-file evaluation
|
||||
* happens in a spawnSync'd bun child rather than a dynamic import.
|
||||
*
|
||||
* Import sites should keep using the ./touchfiles facade, which re-exports
|
||||
* both this module and the data module.
|
||||
*/
|
||||
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
|
||||
import {
|
||||
E2E_TOUCHFILES,
|
||||
E2E_TIERS,
|
||||
LLM_JUDGE_TOUCHFILES,
|
||||
GLOBAL_TOUCHFILES,
|
||||
} from './touchfiles-data';
|
||||
|
||||
/** Repo-relative path of the pure-data file (the map-diff subject). */
|
||||
export const TOUCHFILES_DATA_PATH = 'test/helpers/touchfiles-data.ts';
|
||||
|
||||
// --- Glob matching ---
|
||||
|
||||
/**
|
||||
* Match a file path against a glob pattern.
|
||||
* Supports:
|
||||
* ** — match any number of path segments
|
||||
* * — match within a single segment (no /)
|
||||
*/
|
||||
export function matchGlob(file: string, pattern: string): boolean {
|
||||
const regexStr = pattern
|
||||
.replace(/\./g, '\\.')
|
||||
.replace(/\*\*/g, '{{GLOBSTAR}}')
|
||||
.replace(/\*/g, '[^/]*')
|
||||
.replace(/\{\{GLOBSTAR\}\}/g, '.*');
|
||||
return new RegExp(`^${regexStr}$`).test(file);
|
||||
}
|
||||
|
||||
// --- Base branch detection ---
|
||||
|
||||
/**
|
||||
* Detect the base branch by trying refs in order.
|
||||
* Returns the first valid ref, or null if none found.
|
||||
*/
|
||||
export function detectBaseBranch(cwd: string): string | null {
|
||||
for (const ref of ['origin/main', 'origin/master', 'main', 'master']) {
|
||||
const result = spawnSync('git', ['rev-parse', '--verify', ref], {
|
||||
cwd, stdio: 'pipe', timeout: 3000,
|
||||
});
|
||||
if (result.status === 0) return ref;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a git command and return stdout. FAIL-CLOSED: any failure (spawn
|
||||
* error, non-zero exit) throws — a broken git environment must abort the
|
||||
* suite loudly instead of silently degrading into a full (paid) run.
|
||||
*/
|
||||
function runGitOrThrow(args: string[], cwd: string, spawnImpl: typeof spawnSync): string {
|
||||
const result = spawnImpl('git', args, {
|
||||
cwd, stdio: 'pipe', timeout: 10000, maxBuffer: 8 * 1024 * 1024,
|
||||
});
|
||||
if (result.error || result.status !== 0) {
|
||||
const stderr = result.stderr?.toString().trim() || result.error?.message || 'unknown error';
|
||||
throw new Error(
|
||||
`getChangedFiles: \`git ${args.join(' ')}\` failed in ${cwd} `
|
||||
+ `(exit ${result.status ?? 'spawn-error'}): ${stderr}\n`
|
||||
+ 'Diff-based test selection cannot proceed. Fix the git environment, '
|
||||
+ 'or set EVALS_ALL=1 to deliberately run the full suite.',
|
||||
);
|
||||
}
|
||||
return result.stdout.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the list of files changed relative to the base branch, INCLUDING
|
||||
* uncommitted work. Union of three sources, deduped:
|
||||
* 1. committed: `git diff --name-only <base>...HEAD`
|
||||
* 2. staged + unstaged: `git diff --name-only HEAD`
|
||||
* 3. untracked: `git status --porcelain --untracked-files=all` ('?? ' lines)
|
||||
*
|
||||
* Without 2 and 3, an agent that edits files and runs evals BEFORE
|
||||
* committing gets an empty diff → run-all → the full paid suite every time.
|
||||
*
|
||||
* An empty UNION still means "no changes" and callers keep their intentional
|
||||
* run-all semantics for it (main-branch / periodic full runs depend on that).
|
||||
*
|
||||
* Git failures THROW (see runGitOrThrow) instead of returning [] — the old
|
||||
* behavior made a broken git environment indistinguishable from a clean tree.
|
||||
*
|
||||
* `spawnImpl` is injectable for tests.
|
||||
*/
|
||||
export function getChangedFiles(
|
||||
baseBranch: string,
|
||||
cwd: string,
|
||||
spawnImpl: typeof spawnSync = spawnSync,
|
||||
): string[] {
|
||||
// core.quotePath=false: without it git C-escapes non-ASCII bytes
|
||||
// ("docs/r\303\251sum\303\251.md"), the escaped string matches no glob,
|
||||
// and the dependent test is silently DESELECTED — under-selection, the
|
||||
// exact direction selection must fail away from.
|
||||
const noQuote = ['-c', 'core.quotePath=false'];
|
||||
const committed = runGitOrThrow([...noQuote, 'diff', '--name-only', `${baseBranch}...HEAD`], cwd, spawnImpl)
|
||||
.trim().split('\n').filter(Boolean);
|
||||
const uncommitted = runGitOrThrow([...noQuote, 'diff', '--name-only', 'HEAD'], cwd, spawnImpl)
|
||||
.trim().split('\n').filter(Boolean);
|
||||
const untracked = runGitOrThrow([...noQuote, 'status', '--porcelain', '--untracked-files=all'], cwd, spawnImpl)
|
||||
.split('\n')
|
||||
.filter(line => line.startsWith('?? '))
|
||||
.map(line => {
|
||||
let p = line.slice(3);
|
||||
// residual quoting (embedded quote/newline) — strip the wrapper
|
||||
if (p.startsWith('"') && p.endsWith('"')) p = p.slice(1, -1);
|
||||
return p;
|
||||
});
|
||||
return [...new Set([...committed, ...uncommitted, ...untracked])];
|
||||
}
|
||||
|
||||
// --- Touchfile map diffing ---
|
||||
|
||||
/** The four exports of touchfiles-data.ts, as plain data. */
|
||||
export interface TouchfileMaps {
|
||||
E2E_TOUCHFILES: Record<string, string[]>;
|
||||
E2E_TIERS: Record<string, string>;
|
||||
LLM_JUDGE_TOUCHFILES: Record<string, string[]>;
|
||||
GLOBAL_TOUCHFILES: string[];
|
||||
}
|
||||
|
||||
export type MapDiffCause =
|
||||
| 'missing-base-ref'
|
||||
| 'git-show-failed'
|
||||
| 'import-failed'
|
||||
| 'shape-mismatch';
|
||||
|
||||
export type MapDiffOutcome =
|
||||
| {
|
||||
ok: true;
|
||||
/** Tests whose entry was added, dep-list changed, or tier flipped. */
|
||||
changedTests: string[];
|
||||
/** Keys present in the old maps but gone from every new map (reported, not selected). */
|
||||
removedTests: string[];
|
||||
/** True when the GLOBAL_TOUCHFILES set itself changed — not attributable to any test. */
|
||||
globalTouchfilesChanged: boolean;
|
||||
}
|
||||
| { ok: false; cause: MapDiffCause };
|
||||
|
||||
/** Current maps as a TouchfileMaps value (the "new" side of the diff). */
|
||||
const CURRENT_MAPS: TouchfileMaps = {
|
||||
E2E_TOUCHFILES,
|
||||
E2E_TIERS,
|
||||
LLM_JUDGE_TOUCHFILES,
|
||||
GLOBAL_TOUCHFILES,
|
||||
};
|
||||
|
||||
function isStringArray(v: unknown): v is string[] {
|
||||
return Array.isArray(v) && v.every(x => typeof x === 'string');
|
||||
}
|
||||
|
||||
function isRecordOfStringArrays(v: unknown): v is Record<string, string[]> {
|
||||
return !!v && typeof v === 'object' && !Array.isArray(v)
|
||||
&& Object.values(v).every(isStringArray);
|
||||
}
|
||||
|
||||
function isRecordOfStrings(v: unknown): v is Record<string, string> {
|
||||
return !!v && typeof v === 'object' && !Array.isArray(v)
|
||||
&& Object.values(v).every(x => typeof x === 'string');
|
||||
}
|
||||
|
||||
function isTouchfileMaps(v: unknown): v is TouchfileMaps {
|
||||
if (!v || typeof v !== 'object') return false;
|
||||
const o = v as Record<string, unknown>;
|
||||
return isRecordOfStringArrays(o.E2E_TOUCHFILES)
|
||||
&& isRecordOfStrings(o.E2E_TIERS)
|
||||
&& isRecordOfStringArrays(o.LLM_JUDGE_TOUCHFILES)
|
||||
&& isStringArray(o.GLOBAL_TOUCHFILES);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure map-diff core (injectable for tests — no git, no filesystem).
|
||||
*
|
||||
* A key counts as CHANGED when it was added to any per-key map, its dep-list
|
||||
* array differs, or its tier value flipped. A key counts as REMOVED only when
|
||||
* it is gone from every new per-key map; a key dropped from one map but still
|
||||
* present in another (e.g. tier entry deleted, touchfile entry kept) counts
|
||||
* as changed — conservative, because the test still exists with a different
|
||||
* configuration. GLOBAL_TOUCHFILES is compared as a set; a change there is
|
||||
* not attributable to any test and is flagged for the caller to treat as
|
||||
* "run all".
|
||||
*/
|
||||
export function diffTouchfileMapsCore(
|
||||
oldMaps: TouchfileMaps,
|
||||
newMaps: TouchfileMaps,
|
||||
): { changedTests: string[]; removedTests: string[]; globalTouchfilesChanged: boolean } {
|
||||
const perKeyMapNames = ['E2E_TOUCHFILES', 'E2E_TIERS', 'LLM_JUDGE_TOUCHFILES'] as const;
|
||||
const changed = new Set<string>();
|
||||
const rawRemoved = new Set<string>();
|
||||
|
||||
for (const mapName of perKeyMapNames) {
|
||||
const oldMap: Record<string, unknown> = oldMaps[mapName] ?? {};
|
||||
const newMap: Record<string, unknown> = newMaps[mapName] ?? {};
|
||||
for (const key of Object.keys(newMap)) {
|
||||
if (!(key in oldMap)) {
|
||||
changed.add(key); // added
|
||||
} else if (JSON.stringify(oldMap[key]) !== JSON.stringify(newMap[key])) {
|
||||
changed.add(key); // dep-list edited or tier flipped
|
||||
}
|
||||
}
|
||||
for (const key of Object.keys(oldMap)) {
|
||||
if (!(key in newMap)) rawRemoved.add(key);
|
||||
}
|
||||
}
|
||||
|
||||
const removed = new Set<string>();
|
||||
for (const key of rawRemoved) {
|
||||
const stillExists = perKeyMapNames.some(m => key in (newMaps[m] ?? {}));
|
||||
if (stillExists) changed.add(key);
|
||||
else removed.add(key);
|
||||
}
|
||||
|
||||
const sortedSet = (arr: string[]) => JSON.stringify([...arr].sort());
|
||||
const globalTouchfilesChanged =
|
||||
sortedSet(oldMaps.GLOBAL_TOUCHFILES ?? []) !== sortedSet(newMaps.GLOBAL_TOUCHFILES ?? []);
|
||||
|
||||
return {
|
||||
changedTests: [...changed].sort(),
|
||||
removedTests: [...removed].sort(),
|
||||
globalTouchfilesChanged,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the OLD touchfiles-data.ts from git and diff it against the current
|
||||
* maps. Synchronous: the old file is written to a temp dir and evaluated in
|
||||
* a spawnSync'd bun child that prints the four maps as JSON (module-scope
|
||||
* callers like e2e-helpers.ts cannot await).
|
||||
*
|
||||
* FAIL-CLOSED: every failure returns `{ ok: false, cause }` and the caller
|
||||
* must treat that as "data change is global — run all tests".
|
||||
*
|
||||
* `newMaps` is injectable so integration tests can diff a temp repo's old
|
||||
* version against a fixture instead of this repo's live maps.
|
||||
*/
|
||||
export function diffTouchfileMaps(
|
||||
baseRef: string,
|
||||
cwd: string,
|
||||
newMaps: TouchfileMaps = CURRENT_MAPS,
|
||||
): MapDiffOutcome {
|
||||
try {
|
||||
const verify = spawnSync('git', ['rev-parse', '--verify', baseRef], {
|
||||
cwd, stdio: 'pipe', timeout: 3000,
|
||||
});
|
||||
if (verify.status !== 0) return { ok: false, cause: 'missing-base-ref' };
|
||||
|
||||
const show = spawnSync('git', ['show', `${baseRef}:${TOUCHFILES_DATA_PATH}`], {
|
||||
cwd, stdio: 'pipe', timeout: 5000, maxBuffer: 8 * 1024 * 1024,
|
||||
});
|
||||
if (show.status !== 0) return { ok: false, cause: 'git-show-failed' };
|
||||
const oldSource = show.stdout.toString();
|
||||
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'touchfiles-map-diff-'));
|
||||
try {
|
||||
const dataPath = path.join(tempDir, 'touchfiles-data.ts');
|
||||
fs.writeFileSync(dataPath, oldSource);
|
||||
const loaderPath = path.join(tempDir, 'load-maps.ts');
|
||||
fs.writeFileSync(loaderPath, [
|
||||
`const m = await import(${JSON.stringify(dataPath)});`,
|
||||
'console.log(JSON.stringify({',
|
||||
' E2E_TOUCHFILES: m.E2E_TOUCHFILES,',
|
||||
' E2E_TIERS: m.E2E_TIERS,',
|
||||
' LLM_JUDGE_TOUCHFILES: m.LLM_JUDGE_TOUCHFILES,',
|
||||
' GLOBAL_TOUCHFILES: m.GLOBAL_TOUCHFILES,',
|
||||
'}));',
|
||||
'',
|
||||
].join('\n'));
|
||||
|
||||
// process.execPath is the bun binary when running under bun.
|
||||
const run = spawnSync(process.execPath, ['run', loaderPath], {
|
||||
stdio: 'pipe', timeout: 20000, maxBuffer: 8 * 1024 * 1024,
|
||||
});
|
||||
if (run.status !== 0) return { ok: false, cause: 'import-failed' };
|
||||
|
||||
let oldMaps: unknown;
|
||||
try {
|
||||
oldMaps = JSON.parse(run.stdout.toString());
|
||||
} catch {
|
||||
return { ok: false, cause: 'import-failed' };
|
||||
}
|
||||
if (!isTouchfileMaps(oldMaps)) return { ok: false, cause: 'shape-mismatch' };
|
||||
|
||||
return { ok: true, ...diffTouchfileMapsCore(oldMaps, newMaps) };
|
||||
} finally {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
} catch {
|
||||
// Unexpected failure anywhere in the pipeline (temp dir, git, child
|
||||
// process) — same fail-closed contract as an evaluation failure.
|
||||
return { ok: false, cause: 'import-failed' };
|
||||
}
|
||||
}
|
||||
|
||||
// --- Test selection ---
|
||||
|
||||
/**
|
||||
* Select tests to run based on changed files.
|
||||
*
|
||||
* Algorithm:
|
||||
* 1. If any changed file (other than touchfiles-data.ts) matches a global
|
||||
* touchfile → run ALL tests
|
||||
* 2. If touchfiles-data.ts changed → map-diff it against the base ref and
|
||||
* select only the tests whose map entries changed (fail-closed: any
|
||||
* map-diff failure runs ALL tests, with the cause in the reason string)
|
||||
* 3. For each test, check if any other changed file matches its patterns
|
||||
* 4. Return selected + skipped lists with reason (union of 2 and 3)
|
||||
*
|
||||
* `opts.baseRef` / `opts.cwd` scope the map-diff; they default to
|
||||
* EVALS_BASE || detectBaseBranch || 'main' and the repo root — the same
|
||||
* resolution the module-scope callers (e2e-helpers.ts et al.) use to compute
|
||||
* `changedFiles`, so the two sides of the diff stay consistent.
|
||||
* `opts.mapDiff` injects a precomputed outcome (for tests).
|
||||
*/
|
||||
export function selectTests(
|
||||
changedFiles: string[],
|
||||
touchfiles: Record<string, string[]>,
|
||||
globalTouchfiles: string[] = GLOBAL_TOUCHFILES,
|
||||
opts: { baseRef?: string; cwd?: string; mapDiff?: MapDiffOutcome } = {},
|
||||
): { selected: string[]; skipped: string[]; reason: string; removedTests?: string[] } {
|
||||
const allTestNames = Object.keys(touchfiles);
|
||||
const dataChanged = changedFiles.includes(TOUCHFILES_DATA_PATH);
|
||||
|
||||
// Global touchfile hit → run all. touchfiles-data.ts is excluded here —
|
||||
// its changes route through map-diff below instead of a global run-all.
|
||||
for (const file of changedFiles) {
|
||||
if (file === TOUCHFILES_DATA_PATH) continue;
|
||||
if (globalTouchfiles.some(g => matchGlob(file, g))) {
|
||||
return { selected: allTestNames, skipped: [], reason: `global: ${file}` };
|
||||
}
|
||||
}
|
||||
|
||||
// Map-diff path for data-file changes
|
||||
let mapDiffSelected: Set<string> | null = null;
|
||||
let removedTests: string[] | undefined;
|
||||
if (dataChanged) {
|
||||
const cwd = opts.cwd ?? path.resolve(import.meta.dir, '..', '..');
|
||||
const baseRef = opts.baseRef
|
||||
|| process.env.EVALS_BASE
|
||||
|| detectBaseBranch(cwd)
|
||||
|| 'main';
|
||||
const outcome = opts.mapDiff ?? diffTouchfileMaps(baseRef, cwd);
|
||||
if (!outcome.ok) {
|
||||
return {
|
||||
selected: allTestNames,
|
||||
skipped: [],
|
||||
reason: `global — touchfiles-data changed (${outcome.cause})`,
|
||||
};
|
||||
}
|
||||
if (outcome.globalTouchfilesChanged) {
|
||||
return {
|
||||
selected: allTestNames,
|
||||
skipped: [],
|
||||
reason: 'global — touchfiles-data changed (GLOBAL_TOUCHFILES edited)',
|
||||
};
|
||||
}
|
||||
// Scope to this map's keys (E2E and LLM-judge selections run separately).
|
||||
mapDiffSelected = new Set(outcome.changedTests.filter(t => t in touchfiles));
|
||||
removedTests = outcome.removedTests;
|
||||
}
|
||||
|
||||
// Per-test matching for the remaining changed files
|
||||
const otherFiles = changedFiles.filter(f => f !== TOUCHFILES_DATA_PATH);
|
||||
const selected: string[] = [];
|
||||
const skipped: string[] = [];
|
||||
for (const [testName, patterns] of Object.entries(touchfiles)) {
|
||||
const hit = otherFiles.some(f => patterns.some(p => matchGlob(f, p)))
|
||||
|| (mapDiffSelected !== null && mapDiffSelected.has(testName));
|
||||
(hit ? selected : skipped).push(testName);
|
||||
}
|
||||
|
||||
if (dataChanged) {
|
||||
return { selected, skipped, reason: 'map-diff', removedTests };
|
||||
}
|
||||
return { selected, skipped, reason: 'diff' };
|
||||
}
|
||||
@@ -0,0 +1,813 @@
|
||||
/**
|
||||
* Touchfile maps — the DATA half of diff-based test selection.
|
||||
*
|
||||
* LITERALS ONLY. This file must contain zero import statements and zero
|
||||
* executable logic: no function calls, no spreads, no template literals —
|
||||
* just string / array / Record literals. That property is load-bearing:
|
||||
* map-diff selection evaluates OLD git versions of this file standalone to
|
||||
* diff the maps across commits, which only works while the file stays pure,
|
||||
* importable data. test/touchfiles-facade.test.ts enforces this with a
|
||||
* comment-and-string-stripping tripwire.
|
||||
*
|
||||
* The selection logic (matchGlob, detectBaseBranch, getChangedFiles,
|
||||
* selectTests) lives in ./test-selection.ts. Import sites should keep using
|
||||
* the ./touchfiles facade, which re-exports both halves.
|
||||
*/
|
||||
|
||||
// --- Touchfile maps ---
|
||||
|
||||
/**
|
||||
* E2E test touchfiles — keyed by testName (the string passed to runSkillTest).
|
||||
* Each test lists the file patterns that, if changed, require the test to run.
|
||||
*/
|
||||
export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// Browse core (+ test-server dependency)
|
||||
'browse-basic': ['browse/src/**', 'browse/test/test-server.ts'],
|
||||
'browse-snapshot': ['browse/src/**', 'browse/test/test-server.ts'],
|
||||
|
||||
// Hermetic isolation canaries (hermetic-env.ts is also a GLOBAL touchfile;
|
||||
// these entries exist so the canaries themselves stay tier-classified)
|
||||
'hermetic-canary': ['test/helpers/hermetic-env.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-hermetic-canary.test.ts', 'lib/conductor-env-shim.ts'],
|
||||
'hermetic-sentinel': ['test/helpers/hermetic-env.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-hermetic-canary.test.ts', 'lib/conductor-env-shim.ts'],
|
||||
|
||||
// P4 first-run scaffold (activation lift) — the detection binary end-to-end
|
||||
// through the real runner, plus the preamble wiring that gates + maps it.
|
||||
'first-task-scaffold': ['bin/gstack-first-task-detect', 'scripts/resolvers/preamble/generate-first-run-guidance.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'test/skill-e2e-first-task-scaffold.test.ts', 'test/helpers/session-runner.ts'],
|
||||
|
||||
// SKILL.md setup + preamble (depend on ROOT SKILL.md + gen-skill-docs)
|
||||
'skillmd-setup-discovery': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'skillmd-no-local-binary': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'skillmd-outside-git': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
'session-awareness': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'operational-learning': ['scripts/resolvers/preamble.ts', 'bin/gstack-learnings-log'],
|
||||
|
||||
// QA (+ test-server dependency)
|
||||
'qa-quick': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts'],
|
||||
'qa-b6-static': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval.html', 'test/fixtures/qa-eval-ground-truth.json'],
|
||||
'qa-b7-spa': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-spa.html', 'test/fixtures/qa-eval-spa-ground-truth.json'],
|
||||
'qa-b8-checkout': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-checkout.html', 'test/fixtures/qa-eval-checkout-ground-truth.json'],
|
||||
'qa-only-no-fix': ['qa-only/**', 'qa/templates/**'],
|
||||
'qa-fix-loop': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts'],
|
||||
'qa-bootstrap': ['qa/**', 'ship/**'],
|
||||
|
||||
// Review
|
||||
'review-sql-injection': ['review/**', 'test/fixtures/review-eval-vuln.rb', 'test/skill-e2e-review.test.ts'],
|
||||
'review-enum-completeness': ['review/**', 'test/fixtures/review-eval-enum*.rb', 'test/skill-e2e-review.test.ts'],
|
||||
'review-base-branch': ['review/**', 'test/skill-e2e-review-attribution.test.ts'],
|
||||
'review-design-lite': ['review/**', 'test/fixtures/review-eval-design-slop.*', 'test/skill-e2e-review.test.ts'],
|
||||
|
||||
// Review Army (specialist dispatch)
|
||||
'review-army-migration-safety': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope'],
|
||||
'review-army-perf-n-plus-one': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope'],
|
||||
'review-army-delivery-audit': ['review/**', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-quality-score': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-json-findings': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-red-team': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-consensus': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
|
||||
// Office Hours
|
||||
'office-hours-spec-review': ['office-hours/**', 'scripts/gen-skill-docs.ts'],
|
||||
'office-hours-forcing-energy': ['office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts'],
|
||||
'office-hours-builder-wildness': ['office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts'],
|
||||
|
||||
// Plan reviews
|
||||
'plan-ceo-review': ['plan-ceo-review/**'],
|
||||
'plan-ceo-review-selective': ['plan-ceo-review/**'],
|
||||
'plan-ceo-review-benefits': ['plan-ceo-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
'plan-ceo-review-expansion-energy': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts'],
|
||||
'plan-eng-review': ['plan-eng-review/**'],
|
||||
'plan-eng-review-artifact': ['plan-eng-review/**'],
|
||||
'plan-review-report': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Plan-mode smoke tests — gate-tier safety regression tests. Each test file
|
||||
// contains TWO test cases as of v1.21: the baseline plan-mode case and the
|
||||
// AskUserQuestion-blocked regression case (--disallowedTools AskUserQuestion
|
||||
// parameterized — the flag set Conductor uses by default). Touchfiles
|
||||
// include question-tuning.ts and generate-ask-user-format.ts because the
|
||||
// AUTO_DECIDE preamble injection lives there and changes can flip the
|
||||
// regression test outcome between 'asked' and 'auto_decided'.
|
||||
'plan-ceo-review-plan-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'],
|
||||
'plan-eng-review-plan-mode': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'],
|
||||
'plan-design-review-plan-mode': ['plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts'],
|
||||
'plan-devex-review-plan-mode': ['plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts'],
|
||||
// Covers ceo (preamble misfire) + eng/design (scope-gate bypass must not
|
||||
// fire outside plan mode) + the named-target exception case. 4 PTY runs;
|
||||
// in CI these run CONCURRENT with the rest of the pty-plan-smoke suite
|
||||
// (--max-concurrency + --retry 1), so worst-case cost is ~2x a single
|
||||
// pass of each, sharing the API budget with sibling tests — not the
|
||||
// sequential ~+10min a local read suggests.
|
||||
'plan-mode-no-op': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-mode-no-op.test.ts'],
|
||||
|
||||
// v1.21+ AskUserQuestion-blocked regression tests — Conductor launches
|
||||
// claude with `--disallowedTools AskUserQuestion --permission-mode default`
|
||||
// (verified via `ps`); skills must still surface user-decisions through a
|
||||
// fallback path (mcp__conductor__AskUserQuestion or plan-file flow) rather
|
||||
// than silently auto-deciding. Parameterized regression test cases live
|
||||
// INSIDE the existing 4 plan-X-review-plan-mode test files (covered
|
||||
// transitively by the entries above). Two new standalone files exist for
|
||||
// skills with no prior plan-mode test:
|
||||
'office-hours-auto-mode': ['office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-office-hours-auto-mode.test.ts'],
|
||||
'office-hours-phase4-fork': ['office-hours/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/question-tuning.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours-phase4.test.ts'],
|
||||
'llm-judge-recommendation': ['test/helpers/llm-judge.ts', 'test/llm-judge-recommendation.test.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'codex/SKILL.md.tmpl', 'scripts/resolvers/review.ts'],
|
||||
// v1.21+ AUTO_DECIDE preserve eval (periodic). Verifies the Tool resolution
|
||||
// fix doesn't trip the legitimate /plan-tune opt-in path: when the user has
|
||||
// written a never-ask preference, AUQ should still auto-decide rather than
|
||||
// surfacing the question. Touches the question-tuning + preference
|
||||
// infrastructure plus the resolvers that own the AUTO_DECIDE preamble.
|
||||
'auto-decide-preserved': ['scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
|
||||
// Conductor → prose decision brief (Conductor signal makes prose the default;
|
||||
// the PreToolUse hook denies the flaky tool). Touches the resolver that owns
|
||||
// the Conductor rule, the preamble signal, the hook, and the detection helper.
|
||||
'conductor-prose': ['scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble.ts', 'plan-eng-review/**', 'hosts/claude/hooks/question-preference-hook.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-conductor-prose.test.ts'],
|
||||
|
||||
// Real-PTY E2E batch (#6 new tests on the harness).
|
||||
// Each one tests behavior the SDK harness can't observe (rendered TTY,
|
||||
// numbered-option lists, multi-phase ordering, idempotency state echo).
|
||||
'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts'],
|
||||
'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'],
|
||||
'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'],
|
||||
'budget-regression-pty': ['test/helpers/eval-store.ts', 'test/skill-budget-regression.test.ts'],
|
||||
'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-ship-idempotency.test.ts'],
|
||||
'ship-section-loading': ['ship/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-ship-section-loading.test.ts'],
|
||||
'plan-ceo-section-loading': ['plan-ceo-review/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
|
||||
// Data-driven behavioral guard for the 'plan'/'prompt' carves (eng, design,
|
||||
// devex, office-hours + future PR2 carves). One file iterating CARVE_GUARDS;
|
||||
// the selector sets GSTACK_CARVE_SKILL=<name> to scope cost to the changed
|
||||
// skill (D-CODEX A). Touching the registry/helper or sections.ts runs all.
|
||||
'carve-section-loading': ['plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
|
||||
'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-autoplan-chain.test.ts'],
|
||||
'e2e-harness-audit': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
|
||||
// Per-finding AskUserQuestion count + review-report-at-bottom assertion.
|
||||
// Each test drives its skill end-to-end; touchfiles include preamble +
|
||||
// completion-status resolvers because they affect question cadence and
|
||||
// terminal output (the regression surface this test catches).
|
||||
'plan-ceo-finding-count': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'],
|
||||
'plan-eng-finding-count': ['plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'],
|
||||
'plan-design-finding-count': ['plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'],
|
||||
'plan-devex-finding-count': ['plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'],
|
||||
|
||||
// Gate-tier reviewCount-floor counterparts. Catch the May 2026 transcript
|
||||
// bug (model wrote a plan-mode plan and ExitPlanMode'd without firing any
|
||||
// review-phase AskUserQuestion). Uses runPlanSkillFloorCheck — minimal
|
||||
// "did agent fire ANY AUQ?" observer that exits early on first non-permission
|
||||
// numbered-option render. ~1-3 min typical wall time per test, ~$2-6 total.
|
||||
'plan-eng-finding-floor': ['plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-finding-floor.test.ts'],
|
||||
'plan-ceo-finding-floor': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-finding-floor.test.ts'],
|
||||
'plan-design-finding-floor': ['plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-design-finding-floor.test.ts'],
|
||||
'plan-devex-finding-floor': ['plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-devex-finding-floor.test.ts'],
|
||||
|
||||
// Multi-finding batching regression — periodic tier complement to the
|
||||
// gate-tier finding-floor. Catches the May 2026 transcript shape where
|
||||
// a model fires one AUQ then batches the rest into a "## Decisions to
|
||||
// confirm" plan write. runPlanSkillFloorCheck cannot detect that shape
|
||||
// (it exits on first AUQ); runPlanSkillCounting can.
|
||||
'plan-eng-multi-finding-batching': ['plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'],
|
||||
'plan-ceo-split-overflow': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'],
|
||||
'brain-privacy-gate': ['scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-brain-privacy-gate.test.ts'],
|
||||
|
||||
// /setup-gbrain Path 4 (Remote MCP) — happy + bad-token end-to-end via
|
||||
// Agent SDK. Gate-tier (deterministic stub server, fixed inputs); fires
|
||||
// when the skill template, the verify helper, the artifacts-init helper,
|
||||
// or the detect script changes.
|
||||
'setup-gbrain-remote': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'bin/gstack-artifacts-init', 'bin/gstack-gbrain-detect', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-setup-gbrain-remote.test.ts'],
|
||||
'setup-gbrain-bad-token': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-setup-gbrain-bad-token.test.ts'],
|
||||
// v1.34.0.0 split-engine Path 4 + Step 4.5 Yes (local PGLite for code).
|
||||
// Periodic-tier per codex #12 (AgentSDK harness is non-deterministic).
|
||||
// Fires when the setup-gbrain template, install/verify/init helpers, or
|
||||
// the agent-sdk-runner harness changes.
|
||||
'setup-gbrain-path4-local-pglite': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'bin/gstack-gbrain-install', 'bin/gstack-gbrain-detect', 'lib/gbrain-local-status.ts', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts'],
|
||||
|
||||
// AskUserQuestion format regression (RECOMMENDATION + Completeness: N/10)
|
||||
// Fires when either template OR the two preamble resolvers change.
|
||||
'plan-ceo-review-format-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'],
|
||||
'plan-ceo-review-format-approach': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'],
|
||||
'plan-eng-review-format-coverage': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'],
|
||||
'plan-eng-review-format-kind': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'],
|
||||
|
||||
// v1.7.0.0 Pros/Cons format cadence + format + negative-escape evals.
|
||||
// Dependencies: same as format-mode + the 4 plan-review templates + overlay.
|
||||
// All periodic-tier (non-deterministic Opus 4.7 behavior).
|
||||
'plan-ceo-review-prosons-cadence': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'plan-review-prosons-format': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'plan-review-prosons-hardstop-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'plan-review-prosons-neutral-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
|
||||
// Expanded coverage (CT3) — 6 non-plan-review skills inherit Pros/Cons via preamble
|
||||
'ship-prosons-format': ['ship/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'office-hours-prosons-format': ['office-hours/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'investigate-prosons-format': ['investigate/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'qa-prosons-format': ['qa/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'review-prosons-format': ['review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'design-review-prosons-format': ['design-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'document-release-prosons-format': ['document-release/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
|
||||
// /plan-tune (v1 observational)
|
||||
'plan-tune-inspect': ['plan-tune/**', 'scripts/question-registry.ts', 'scripts/psychographic-signals.ts', 'scripts/one-way-doors.ts', 'bin/gstack-question-log', 'bin/gstack-question-preference', 'bin/gstack-developer-profile'],
|
||||
|
||||
// /plan-tune cathedral (T16 — 5 E2E scenarios, all gate per D12)
|
||||
'plan-tune-hook-capture': ['hosts/claude/hooks/**', 'bin/gstack-question-log', 'bin/gstack-developer-profile', 'plan-tune/**'],
|
||||
'plan-tune-enforcement': ['hosts/claude/hooks/**', 'bin/gstack-question-preference', 'scripts/question-registry.ts'],
|
||||
'plan-tune-annotation': ['hosts/claude/hooks/**', 'scripts/declared-annotation.ts', 'scripts/psychographic-signals.ts', 'scripts/question-registry.ts'],
|
||||
'plan-tune-codex-import': ['bin/gstack-codex-session-import', 'bin/gstack-question-log', 'docs/spikes/codex-session-format.md'],
|
||||
'plan-tune-dream-cycle': ['bin/gstack-distill-free-text', 'bin/gstack-distill-apply', 'hosts/claude/hooks/**', 'plan-tune/**'],
|
||||
|
||||
// Codex offering verification
|
||||
'codex-offered-office-hours': ['office-hours/**', 'scripts/gen-skill-docs.ts'],
|
||||
'codex-offered-ceo-review': ['plan-ceo-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
'codex-offered-design-review': ['plan-design-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
'codex-offered-eng-review': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Ship
|
||||
'ship-base-branch': ['ship/**', 'bin/gstack-repo-mode', 'test/skill-e2e-review-attribution.test.ts'],
|
||||
'ship-local-workflow': ['ship/**', 'scripts/gen-skill-docs.ts'],
|
||||
'review-dashboard-via': ['ship/**', 'scripts/resolvers/review.ts', 'codex/**', 'autoplan/**', 'land-and-deploy/**', 'test/skill-e2e-review-attribution.test.ts'],
|
||||
|
||||
// Retro
|
||||
'retro': ['retro/**', 'test/skill-e2e-retro.test.ts'],
|
||||
'retro-base-branch': ['retro/**', 'test/skill-e2e-retro.test.ts'],
|
||||
|
||||
// Global discover
|
||||
'global-discover': ['bin/gstack-global-discover.ts', 'test/global-discover.test.ts'],
|
||||
|
||||
// CSO
|
||||
'cso-full-audit': ['cso/**'],
|
||||
'cso-diff-mode': ['cso/**'],
|
||||
'cso-infra-scope': ['cso/**'],
|
||||
|
||||
// Learnings
|
||||
'learnings-show': ['learn/**', 'bin/gstack-learnings-search', 'bin/gstack-learnings-log', 'scripts/resolvers/learnings.ts'],
|
||||
|
||||
// Session Intelligence (timeline, context recovery, /context-save + /context-restore)
|
||||
'timeline-event-flow': ['bin/gstack-timeline-log', 'bin/gstack-timeline-read'],
|
||||
'context-recovery-artifacts': ['scripts/resolvers/preamble.ts', 'bin/gstack-timeline-log', 'bin/gstack-slug', 'learn/**'],
|
||||
'context-save-writes-file': ['context-save/**', 'bin/gstack-slug'],
|
||||
'context-restore-loads-latest': ['context-restore/**', 'bin/gstack-slug'],
|
||||
|
||||
// Context skills E2E (live-fire, Skill-tool routing path) — see
|
||||
// test/skill-e2e-context-skills.test.ts. These are periodic-tier because
|
||||
// each one spawns claude -p and costs ~$0.20-$0.40. Collectively they
|
||||
// verify the thing the /checkpoint → /context-save rename was for.
|
||||
'context-save-routing': ['context-save/**', 'scripts/resolvers/preamble.ts'],
|
||||
'context-save-then-restore-roundtrip': ['context-save/**', 'context-restore/**', 'bin/gstack-slug'],
|
||||
'context-restore-fragment-match': ['context-restore/**'],
|
||||
'context-restore-empty-state': ['context-restore/**'],
|
||||
'context-restore-list-delegates': ['context-restore/**'],
|
||||
'context-restore-legacy-compat': ['context-restore/**'],
|
||||
'context-save-list-current-branch': ['context-save/**'],
|
||||
'context-save-list-all-branches': ['context-save/**'],
|
||||
|
||||
// Document-release
|
||||
'document-release': ['document-release/**'],
|
||||
|
||||
// Codex (Claude E2E — tests /codex skill via Claude)
|
||||
'codex-review': ['codex/**'],
|
||||
|
||||
// Codex E2E (tests skills via Codex CLI + worktree)
|
||||
'codex-discover-skill': ['codex/**', '.agents/skills/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts'],
|
||||
'codex-review-findings': ['review/**', '.agents/skills/gstack-review/**', 'codex/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts'],
|
||||
|
||||
// Gemini E2E — smoke test only (Gemini gets lost in worktrees on complex tasks)
|
||||
'gemini-smoke': ['.agents/skills/**', 'test/helpers/gemini-session-runner.ts', 'lib/worktree.ts'],
|
||||
|
||||
|
||||
// Coverage audit (shared fixture) + triage + gates
|
||||
'ship-coverage-audit': ['ship/**', 'test/fixtures/coverage-audit-fixture.ts', 'bin/gstack-repo-mode'],
|
||||
'review-coverage-audit': ['review/**', 'test/fixtures/coverage-audit-fixture.ts', 'test/skill-e2e-coverage-audit.test.ts'],
|
||||
'plan-eng-coverage-audit': ['plan-eng-review/**', 'test/fixtures/coverage-audit-fixture.ts', 'test/skill-e2e-coverage-audit.test.ts'],
|
||||
'ship-triage': ['ship/**', 'bin/gstack-repo-mode', 'test/skill-e2e-triage.test.ts'],
|
||||
|
||||
// Plan completion audit + verification
|
||||
'ship-plan-completion': ['ship/**', 'scripts/gen-skill-docs.ts'],
|
||||
'ship-plan-verification': ['ship/**', 'qa-only/**', 'scripts/gen-skill-docs.ts'],
|
||||
'review-plan-completion': ['review/**', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Design
|
||||
'design-consultation-core': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/helpers/llm-judge.ts'],
|
||||
'design-consultation-existing': ['design-consultation/**', 'scripts/gen-skill-docs.ts'],
|
||||
'design-consultation-research': ['design-consultation/**', 'scripts/gen-skill-docs.ts'],
|
||||
'design-consultation-preview': ['design-consultation/**', 'scripts/gen-skill-docs.ts'],
|
||||
'plan-design-review-no-ui-scope': ['plan-design-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
'design-review-fix': ['design-review/**', 'browse/src/**', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Design Shotgun
|
||||
'design-shotgun-path': ['design-shotgun/**', 'design/src/**', 'scripts/resolvers/design.ts'],
|
||||
'design-shotgun-session': ['design-shotgun/**', 'scripts/resolvers/design.ts'],
|
||||
'design-shotgun-full': ['design-shotgun/**', 'design/src/**', 'browse/src/**'],
|
||||
|
||||
// /diagram (diagram-render bundle consumers). Triplet = deterministic
|
||||
// functional (gate); authoring quality = LLM-judged benchmark (periodic).
|
||||
'diagram-triplet': ['diagram/**', 'lib/diagram-render/**', 'browse/src/write-commands.ts', 'browse/src/read-commands.ts'],
|
||||
'diagram-authoring-quality': ['diagram/**', 'lib/diagram-render/**', 'test/helpers/llm-judge.ts'],
|
||||
|
||||
// gstack-upgrade
|
||||
'gstack-upgrade-happy-path': ['gstack-upgrade/**'],
|
||||
|
||||
// Deploy skills
|
||||
'land-and-deploy-workflow': ['land-and-deploy/**', 'scripts/gen-skill-docs.ts'],
|
||||
'land-and-deploy-first-run': ['land-and-deploy/**', 'scripts/gen-skill-docs.ts', 'bin/gstack-slug'],
|
||||
'land-and-deploy-review-gate': ['land-and-deploy/**', 'bin/gstack-review-read'],
|
||||
'canary-workflow': ['canary/**', 'browse/src/**'],
|
||||
'benchmark-workflow': ['benchmark/**', 'browse/src/**'],
|
||||
'setup-deploy-workflow': ['setup-deploy/**', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
|
||||
// Autoplan
|
||||
'autoplan-core': ['autoplan/**', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**'],
|
||||
'autoplan-dual-voice': ['autoplan/**', 'codex/**', 'bin/gstack-codex-probe', 'scripts/resolvers/review.ts', 'scripts/resolvers/design.ts'],
|
||||
|
||||
// Multi-provider benchmark adapters — live API smoke against real claude/codex/gemini CLIs
|
||||
'benchmark-providers-live': ['bin/gstack-model-benchmark', 'test/helpers/providers/**', 'test/helpers/benchmark-runner.ts', 'test/helpers/pricing.ts', 'test/skill-e2e-benchmark-providers.test.ts'],
|
||||
|
||||
// Browser-skills Phase 2a — /scrape + /skillify (v1.19.0.0). Gate-tier
|
||||
// E2E covers the D1 (provenance guard), D3 (atomic write) contracts plus
|
||||
// the basic loop. Shared deps: both skill templates, the D3 helper, the
|
||||
// Phase 1 runtime, and the bundled hackernews-frontpage reference (the
|
||||
// match-path test relies on it).
|
||||
'scrape-match-path': [
|
||||
'scrape/**', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts',
|
||||
'browser-skills/hackernews-frontpage/**',
|
||||
],
|
||||
'scrape-prototype-path': [
|
||||
'scrape/**', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts',
|
||||
],
|
||||
'skillify-happy-path': [
|
||||
'skillify/**', 'scrape/**', 'browse/src/browser-skill-write.ts',
|
||||
'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts',
|
||||
],
|
||||
'skillify-provenance-refusal': [
|
||||
'skillify/**', 'browse/src/browser-skill-write.ts',
|
||||
],
|
||||
'skillify-approval-reject': [
|
||||
'skillify/**', 'scrape/**', 'browse/src/browser-skill-write.ts',
|
||||
],
|
||||
|
||||
// Skill routing — journey-stage tests (depend on ALL skill descriptions)
|
||||
'journey-ideation': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-plan-eng': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-debug': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-code-review': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-ship': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-docs': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-retro': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-design-system': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-visual-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Opus 4.7 behavior evals — keys match testName: values in the test file.
|
||||
// Routing sub-tests use template literal `routing-${c.name}` testNames,
|
||||
// which the touchfile completeness scanner skips; they inherit selection
|
||||
// from the file-level touchfile entry via GLOBAL_TOUCHFILES.
|
||||
'fanout-arm-overlay-on':
|
||||
['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts'],
|
||||
'fanout-arm-overlay-off':
|
||||
['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts'],
|
||||
|
||||
// Overlay efficacy harness (SDK) — measures whether overlay nudges change
|
||||
// behavior under @anthropic-ai/claude-agent-sdk (closer to real Claude Code
|
||||
// than `claude -p`). testNames in the file are template literals so the
|
||||
// completeness scanner doesn't require them; these entries exist for
|
||||
// diff-based selection accuracy.
|
||||
'overlay-harness-opus-4-7-fanout-toy': [
|
||||
'model-overlays/**',
|
||||
'test/fixtures/overlay-nudges.ts',
|
||||
'test/helpers/agent-sdk-runner.ts',
|
||||
'scripts/resolvers/model-overlay.ts',
|
||||
'test/skill-e2e-overlay-harness.test.ts',
|
||||
],
|
||||
'overlay-harness-opus-4-7-fanout-realistic': [
|
||||
'model-overlays/**',
|
||||
'test/fixtures/overlay-nudges.ts',
|
||||
'test/helpers/agent-sdk-runner.ts',
|
||||
'scripts/resolvers/model-overlay.ts',
|
||||
'test/skill-e2e-overlay-harness.test.ts',
|
||||
],
|
||||
|
||||
// /ios-qa — agent flow E2E. Daemon + stub StateServer + codegen
|
||||
// exercised end-to-end. The no-device path is gate-tier; the with-device
|
||||
// path requires GSTACK_HAS_IOS_DEVICE=1 and is periodic-tier.
|
||||
'ios-qa-e2e': ['ios-qa/**', 'ios-fix/**', 'ios-design-review/**', 'ios-clean/**', 'ios-sync/**', 'test/skill-e2e-ios.test.ts'],
|
||||
// Swift-build invariant test — requires the Swift toolchain. Compiles the
|
||||
// fixture SPM package + runs the XCTest suite that validates the real
|
||||
// Swift StateServer implementation (loopback bind, boot token rotation,
|
||||
// session lock). Periodic-tier — Swift build is heavier than TS unit tests.
|
||||
'ios-qa-swift-build': ['ios-qa/templates/**', 'test/fixtures/ios-qa/FixtureApp/**', 'test/skill-e2e-ios-swift-build.test.ts'],
|
||||
// Real-device path — only runs with GSTACK_HAS_IOS_DEVICE=1 + a paired
|
||||
// iPhone. Validates the CoreDevice agent + iOS SDK toolchain. Periodic-tier.
|
||||
'ios-qa-device': ['ios-qa/templates/**', 'test/fixtures/ios-qa/FixtureApp/**', 'test/skill-e2e-ios-device.test.ts'],
|
||||
|
||||
// /spec end-to-end via PTY — exercises the full Phase 1→5 pipeline
|
||||
// including --execute spawn. Periodic-tier — paid + non-deterministic.
|
||||
'spec-execute': ['spec/**', 'test/skill-e2e-spec-execute.test.ts'],
|
||||
|
||||
// /office-hours brain-writeback path under fake gbrain CLI (v1.50.0.0
|
||||
// T7). Drives /office-hours with a regenerated SKILL.md that has the
|
||||
// compressed GBRAIN_SAVE_RESULTS block + a fake gbrain on PATH; asserts
|
||||
// the agent calls `gbrain put office-hours/<slug>` with valid YAML
|
||||
// frontmatter. Touched by anything that changes resolver output, gen
|
||||
// pipeline, detection helper, refresh subcommand, or the on-demand
|
||||
// docs the resolver points to.
|
||||
'office-hours-brain-writeback': [
|
||||
'scripts/resolvers/gbrain.ts',
|
||||
'scripts/gen-skill-docs.ts',
|
||||
'bin/gstack-gbrain-detect',
|
||||
'bin/gstack-config',
|
||||
'office-hours/SKILL.md.tmpl',
|
||||
'docs/gbrain-write-surfaces.md',
|
||||
'test/fixtures/office-hours-brain-writeback/**',
|
||||
'test/skill-e2e-office-hours-brain-writeback.test.ts',
|
||||
],
|
||||
|
||||
// gbrain CLI real round-trip against a local PGLite store (v1.50.0.0
|
||||
// T11). Proves the gbrain CLI persistence contract gstack relies on —
|
||||
// a `gbrain put` followed by `gbrain get` returns the body. Skips if
|
||||
// VOYAGE_API_KEY is unset OR gbrain CLI not on PATH. Touched by the
|
||||
// resolver (which emits the CLI shape) and the test itself.
|
||||
'gbrain-roundtrip-local': [
|
||||
'scripts/resolvers/gbrain.ts',
|
||||
'test/skill-e2e-gbrain-roundtrip-local.test.ts',
|
||||
],
|
||||
|
||||
};
|
||||
|
||||
/**
|
||||
* E2E test tiers — 'gate' blocks PRs, 'periodic' runs weekly/on-demand.
|
||||
* Must have exactly the same keys as E2E_TOUCHFILES.
|
||||
*/
|
||||
export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
// Browse core — gate (if browse breaks, everything breaks)
|
||||
'browse-basic': 'gate',
|
||||
'browse-snapshot': 'gate',
|
||||
|
||||
// Hermetic isolation — gate (deterministic env/config assertions; if the
|
||||
// clean room breaks, every other eval's signal is contaminated)
|
||||
'hermetic-canary': 'gate',
|
||||
'hermetic-sentinel': 'gate',
|
||||
|
||||
// SKILL.md setup — gate (if setup breaks, no skill works)
|
||||
'skillmd-setup-discovery': 'gate',
|
||||
'skillmd-no-local-binary': 'gate',
|
||||
'skillmd-outside-git': 'gate',
|
||||
'session-awareness': 'gate',
|
||||
'operational-learning': 'gate',
|
||||
|
||||
// P4 first-run scaffold — periodic (onboarding, non-safety, model-touched marker)
|
||||
'first-task-scaffold': 'periodic',
|
||||
|
||||
// QA — gate for functional, periodic for quality/benchmarks
|
||||
'qa-quick': 'gate',
|
||||
'qa-b6-static': 'periodic',
|
||||
'qa-b7-spa': 'periodic',
|
||||
'qa-b8-checkout': 'periodic',
|
||||
'qa-only-no-fix': 'gate', // CRITICAL guardrail: Edit tool forbidden
|
||||
'qa-fix-loop': 'periodic',
|
||||
'qa-bootstrap': 'gate',
|
||||
|
||||
// Review — gate for functional/guardrails, periodic for quality
|
||||
'review-sql-injection': 'gate', // Security guardrail
|
||||
'review-enum-completeness': 'gate',
|
||||
'review-base-branch': 'gate',
|
||||
'review-design-lite': 'periodic', // 4/7 threshold is subjective
|
||||
'review-coverage-audit': 'gate',
|
||||
'review-plan-completion': 'gate',
|
||||
'review-dashboard-via': 'gate',
|
||||
|
||||
// Review Army — gate for core functionality, periodic for multi-specialist
|
||||
'review-army-migration-safety': 'gate', // Specialist activation guardrail
|
||||
'review-army-perf-n-plus-one': 'gate', // Specialist activation guardrail
|
||||
'review-army-delivery-audit': 'gate', // Delivery integrity guardrail
|
||||
'review-army-quality-score': 'gate', // Score computation
|
||||
'review-army-json-findings': 'gate', // JSON schema compliance
|
||||
'review-army-red-team': 'periodic', // Multi-agent coordination
|
||||
'review-army-consensus': 'periodic', // Multi-specialist agreement
|
||||
|
||||
// Office Hours
|
||||
'office-hours-spec-review': 'gate',
|
||||
// Brain-writeback E2E — periodic per cost (claude -p) + non-deterministic
|
||||
// (model interprets the gbrain instruction). Matches nearby
|
||||
// setup-gbrain-path4-* tier classification.
|
||||
'office-hours-brain-writeback': 'periodic',
|
||||
// GBrain CLI round-trip — periodic per Voyage embedding cost (~$0.001/run)
|
||||
// and external-API-dependency (skips cleanly if VOYAGE_API_KEY unset).
|
||||
'gbrain-roundtrip-local': 'periodic',
|
||||
'office-hours-forcing-energy': 'periodic', // D2a demotion 2026-08: posture score, periodic-grade signal (sibling precedent at office-hours-tone)
|
||||
// 'office-hours-builder-wildness' retiered to periodic in v1.32 contributor
|
||||
// wave: this is an LLM-judge creativity score (axis_a ≥4 on a "wildness"
|
||||
// posture). Per CLAUDE.md tier-classification rules, non-deterministic
|
||||
// quality benchmarks belong in periodic, not gate. The wave's +21-line
|
||||
// CJK preamble cascade (#1205) pushed the score from 5/5 → 3/3 on the
|
||||
// same /office-hours BUILDER prompt — same model, same fixture — proving
|
||||
// the bar is sensitive to preamble-byte changes that have nothing to do
|
||||
// with the test's intent (creativity, not preamble compliance).
|
||||
'office-hours-builder-wildness': 'periodic',
|
||||
|
||||
// Plan reviews — gate for cheap functional, periodic for Opus quality
|
||||
'plan-ceo-review': 'periodic',
|
||||
'plan-ceo-review-selective': 'periodic',
|
||||
'plan-ceo-review-benefits': 'gate',
|
||||
'plan-ceo-review-expansion-energy': 'gate', // V1.1 mode-posture regression gate (Opus generator, Sonnet judge)
|
||||
'plan-eng-review': 'periodic',
|
||||
'plan-eng-review-artifact': 'periodic',
|
||||
'plan-eng-coverage-audit': 'gate',
|
||||
'plan-review-report': 'gate',
|
||||
|
||||
// Plan-mode handshake. plan-ceo/plan-devex ask-first reliably (gate-tier);
|
||||
// plan-eng/plan-design run a long explore/audit before their first
|
||||
// AskUserQuestion, so whether they reach a terminal outcome within the 300s
|
||||
// budget hinges on stochastic ask-first compliance (~50-67%/run measured).
|
||||
// Per the "non-deterministic -> periodic" tiering rule they are periodic:
|
||||
// the hardened ask-first gate + the collapsed-form detector lifted them from
|
||||
// always-failing to mostly-passing, but they are not deterministic gates.
|
||||
'plan-ceo-review-plan-mode': 'gate',
|
||||
'plan-eng-review-plan-mode': 'periodic',
|
||||
'plan-design-review-plan-mode': 'periodic',
|
||||
'plan-devex-review-plan-mode': 'gate',
|
||||
'plan-mode-no-op': 'gate',
|
||||
// v1.21+ auto-mode regression tests
|
||||
'office-hours-auto-mode': 'gate',
|
||||
'auto-decide-preserved': 'periodic',
|
||||
'conductor-prose': 'periodic',
|
||||
'e2e-harness-audit': 'gate',
|
||||
|
||||
// Real-PTY E2E batch — tier classification:
|
||||
// gate: cheap, deterministic, run on every PR
|
||||
// periodic: long-running or expensive (>$3/run), run weekly
|
||||
'auq-format-gate': 'gate', // ~$0.50/run, SDK capture, single skill probe
|
||||
'plan-ceo-mode-routing': 'periodic', // ~$3/run, deep navigation through 8-12 prior AskUserQuestions
|
||||
'plan-design-with-ui-scope': 'gate', // ~$0.80/run
|
||||
'budget-regression-pty': 'gate', // free, library-only assertion
|
||||
'ship-idempotency-pty': 'periodic', // ~$3/run, real /ship in plan mode
|
||||
'ship-section-loading': 'periodic', // ~$3/run, real /ship; asserts section reads
|
||||
'plan-ceo-section-loading': 'periodic', // ~$3-5/run, real /plan-ceo-review; asserts section read
|
||||
'carve-section-loading': 'periodic', // ~$1-2/skill, data-driven; GSTACK_CARVE_SKILL scopes to one
|
||||
'autoplan-chain-pty': 'periodic', // ~$8/run, all 3 phases sequential
|
||||
|
||||
// Per-finding count + review-report-at-bottom — periodic because each
|
||||
// run drives a full skill end-to-end (~25 min, ~$5/run). Sequential
|
||||
// execution during calibration; concurrent opt-in only after measured
|
||||
// comparison agrees (plan §D15).
|
||||
'plan-ceo-finding-count': 'periodic',
|
||||
'plan-eng-finding-count': 'periodic',
|
||||
'plan-design-finding-count': 'periodic',
|
||||
'plan-devex-finding-count': 'periodic',
|
||||
'plan-eng-finding-floor': 'periodic', // stochastic ask-first (see plan-mode-handshake note); periodic
|
||||
'plan-ceo-finding-floor': 'gate',
|
||||
'plan-design-finding-floor': 'periodic', // stochastic ask-first (see plan-mode-handshake note); periodic
|
||||
'plan-devex-finding-floor': 'gate',
|
||||
'plan-eng-multi-finding-batching': 'periodic',
|
||||
'plan-ceo-split-overflow': 'periodic',
|
||||
|
||||
// Privacy gate for gstack-brain-sync — periodic (non-deterministic LLM call,
|
||||
// costs ~$0.30-$0.50 per run, not needed on every commit)
|
||||
'brain-privacy-gate': 'periodic',
|
||||
|
||||
// /setup-gbrain Path 4 (Remote MCP) — periodic-tier. The stub HTTP
|
||||
// server is deterministic but the model's interpretation of "follow
|
||||
// Path 4 only" is not — assertions on which steps the model ran are
|
||||
// flaky. The deterministic gate-tier coverage for Path 4 lives in
|
||||
// test/setup-gbrain-path4-structure.test.ts (free, <200ms). These
|
||||
// E2E tests stay available for on-demand verification of the live
|
||||
// model's behavior against a stub MCP server.
|
||||
'setup-gbrain-remote': 'periodic',
|
||||
'setup-gbrain-bad-token': 'periodic',
|
||||
'setup-gbrain-path4-local-pglite': 'periodic',
|
||||
|
||||
// AskUserQuestion format regression — periodic (Opus 4.7 non-deterministic benchmark)
|
||||
'plan-ceo-review-format-mode': 'periodic',
|
||||
'plan-ceo-review-format-approach': 'periodic',
|
||||
'plan-eng-review-format-coverage': 'periodic',
|
||||
'plan-eng-review-format-kind': 'periodic',
|
||||
|
||||
// Office-hours Phase 4 silent-auto-decide regression — periodic (Phase 4
|
||||
// requires the agent to invent 2-3 architectures, more open-ended than the
|
||||
// 4 plan-format cases above). Reclassify to gate if it turns out stable.
|
||||
'office-hours-phase4-fork': 'periodic',
|
||||
// judgeRecommendation rubric sanity (fixture-based, ~$0.04/run via Haiku)
|
||||
'llm-judge-recommendation': 'periodic',
|
||||
|
||||
// v1.7.0.0 Pros/Cons format — cadence + negative-escape evals (all periodic)
|
||||
'plan-ceo-review-prosons-cadence': 'periodic',
|
||||
'plan-review-prosons-format': 'periodic',
|
||||
'plan-review-prosons-hardstop-neg': 'periodic',
|
||||
'plan-review-prosons-neutral-neg': 'periodic',
|
||||
|
||||
// CT3 expanded coverage — non-plan-review skills inheriting Pros/Cons (all periodic)
|
||||
'ship-prosons-format': 'periodic',
|
||||
'office-hours-prosons-format': 'periodic',
|
||||
'investigate-prosons-format': 'periodic',
|
||||
'qa-prosons-format': 'periodic',
|
||||
'review-prosons-format': 'periodic',
|
||||
'design-review-prosons-format': 'periodic',
|
||||
'document-release-prosons-format': 'periodic',
|
||||
|
||||
// /plan-tune — gate (core v1 DX promise: plain-English intent routing)
|
||||
'plan-tune-inspect': 'gate',
|
||||
|
||||
// /plan-tune cathedral (T16 per D12 — all gate)
|
||||
'plan-tune-hook-capture': 'gate',
|
||||
'plan-tune-enforcement': 'gate',
|
||||
'plan-tune-annotation': 'gate',
|
||||
'plan-tune-codex-import': 'gate',
|
||||
'plan-tune-dream-cycle': 'gate',
|
||||
|
||||
// Codex offering verification
|
||||
'codex-offered-office-hours': 'gate',
|
||||
'codex-offered-ceo-review': 'gate',
|
||||
'codex-offered-design-review': 'gate',
|
||||
'codex-offered-eng-review': 'gate',
|
||||
|
||||
// Session Intelligence — gate for data flow, periodic for agent integration
|
||||
'timeline-event-flow': 'gate', // Binary data flow (no LLM needed)
|
||||
'context-recovery-artifacts': 'gate', // Preamble reads seeded artifacts
|
||||
'context-save-writes-file': 'gate', // /context-save writes a file
|
||||
'context-restore-loads-latest': 'gate', // Cross-branch newest-by-filename restore
|
||||
|
||||
// Context skills live-fire — periodic (each test spawns claude -p, ~$0.20-$0.40)
|
||||
'context-save-routing': 'periodic', // Proves /context-save routes via Skill tool
|
||||
'context-save-then-restore-roundtrip': 'periodic', // Full cycle in one session
|
||||
'context-restore-fragment-match': 'periodic', // /context-restore <fragment>
|
||||
'context-restore-empty-state': 'periodic', // Graceful zero-saves message
|
||||
'context-restore-list-delegates': 'periodic', // /context-restore list redirect
|
||||
'context-restore-legacy-compat': 'periodic', // Pre-rename files still load
|
||||
'context-save-list-current-branch': 'periodic', // Default branch filter
|
||||
'context-save-list-all-branches': 'periodic', // --all flag
|
||||
|
||||
// Ship — gate (end-to-end ship path)
|
||||
'ship-base-branch': 'gate',
|
||||
'ship-local-workflow': 'gate',
|
||||
'ship-coverage-audit': 'gate',
|
||||
'ship-triage': 'gate',
|
||||
'ship-plan-completion': 'gate',
|
||||
'ship-plan-verification': 'gate',
|
||||
|
||||
// Retro — gate for cheap branch detection, periodic for full Opus retro
|
||||
'retro': 'periodic',
|
||||
'retro-base-branch': 'gate',
|
||||
|
||||
// Global discover
|
||||
'global-discover': 'gate',
|
||||
|
||||
// CSO — gate for security guardrails, periodic for quality
|
||||
'cso-full-audit': 'periodic', // D2a demotion 2026-08: 250s/$0.57 full audit; cso targeted tests stay gate
|
||||
'cso-diff-mode': 'gate',
|
||||
'cso-infra-scope': 'periodic',
|
||||
|
||||
// Learnings — gate (functional guardrail: seeded learnings must appear)
|
||||
'learnings-show': 'gate',
|
||||
|
||||
// Document-release — gate (CHANGELOG guardrail)
|
||||
'document-release': 'gate',
|
||||
|
||||
// Codex — periodic (Opus, requires codex CLI)
|
||||
'codex-review': 'periodic',
|
||||
|
||||
// Multi-AI — periodic (require external CLIs)
|
||||
'codex-discover-skill': 'periodic',
|
||||
'codex-review-findings': 'periodic',
|
||||
'gemini-smoke': 'periodic',
|
||||
|
||||
// Design — gate for cheap functional, periodic for Opus/quality
|
||||
'design-consultation-core': 'periodic',
|
||||
'design-consultation-existing': 'periodic',
|
||||
'design-consultation-research': 'periodic', // D2a demotion 2026-08: the two most expensive gate tests ($0.91/304s)
|
||||
'design-consultation-preview': 'periodic', // D2a demotion 2026-08 ($0.89/481s)
|
||||
'plan-design-review-no-ui-scope': 'gate',
|
||||
'design-review-fix': 'periodic',
|
||||
'design-shotgun-path': 'gate',
|
||||
'design-shotgun-session': 'gate',
|
||||
'design-shotgun-full': 'periodic',
|
||||
|
||||
// /diagram — triplet is deterministic functional, judge is a quality benchmark
|
||||
'diagram-triplet': 'gate',
|
||||
'diagram-authoring-quality': 'periodic',
|
||||
|
||||
// gstack-upgrade
|
||||
'gstack-upgrade-happy-path': 'gate',
|
||||
|
||||
// Deploy skills
|
||||
'land-and-deploy-workflow': 'gate',
|
||||
'land-and-deploy-first-run': 'gate',
|
||||
'land-and-deploy-review-gate': 'gate',
|
||||
'canary-workflow': 'gate',
|
||||
'benchmark-workflow': 'gate',
|
||||
'setup-deploy-workflow': 'gate',
|
||||
|
||||
|
||||
// Autoplan — periodic (not yet implemented)
|
||||
'autoplan-core': 'periodic',
|
||||
'autoplan-dual-voice': 'periodic',
|
||||
|
||||
// Multi-provider benchmark — periodic (requires external CLIs + auth, paid)
|
||||
'benchmark-providers-live': 'periodic',
|
||||
|
||||
// Browser-skills Phase 2a — gate (D1/D3 contracts must not silently break)
|
||||
'scrape-match-path': 'gate',
|
||||
'scrape-prototype-path': 'gate',
|
||||
'skillify-happy-path': 'gate',
|
||||
'skillify-provenance-refusal': 'gate',
|
||||
'skillify-approval-reject': 'gate',
|
||||
|
||||
// Skill routing — periodic (LLM routing is non-deterministic)
|
||||
'journey-ideation': 'periodic',
|
||||
'journey-plan-eng': 'periodic',
|
||||
'journey-debug': 'periodic',
|
||||
'journey-qa': 'periodic',
|
||||
'journey-code-review': 'periodic',
|
||||
'journey-ship': 'periodic',
|
||||
'journey-docs': 'periodic',
|
||||
'journey-retro': 'periodic',
|
||||
'journey-design-system': 'periodic',
|
||||
'journey-visual-qa': 'periodic',
|
||||
|
||||
// Opus 4.7 overlay evals — periodic (non-deterministic LLM behavior + Opus cost)
|
||||
'fanout-arm-overlay-on': 'periodic',
|
||||
'fanout-arm-overlay-off': 'periodic',
|
||||
|
||||
// Overlay efficacy harness (SDK, paid) — periodic only
|
||||
'overlay-harness-opus-4-7-fanout-toy': 'periodic',
|
||||
'overlay-harness-opus-4-7-fanout-realistic': 'periodic',
|
||||
|
||||
// /ios-qa daemon + codegen — no-device path runs every PR (no hardware
|
||||
// dependency, deterministic). with-device path requires GSTACK_HAS_IOS_DEVICE.
|
||||
'ios-qa-e2e': 'gate',
|
||||
// Swift toolchain only, no device required, but heavier than TS unit tests.
|
||||
'ios-qa-swift-build': 'periodic',
|
||||
// Requires a real connected + paired iPhone. Manual-trigger only.
|
||||
'ios-qa-device': 'periodic',
|
||||
// /spec end-to-end PTY pipeline (paid, non-deterministic — periodic-tier).
|
||||
'spec-execute': 'periodic',
|
||||
};
|
||||
|
||||
/**
|
||||
* LLM-judge test touchfiles — keyed by test description string.
|
||||
*/
|
||||
export const LLM_JUDGE_TOUCHFILES: Record<string, string[]> = {
|
||||
'command reference table': ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts'],
|
||||
'snapshot flags reference': ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts'],
|
||||
'browse/SKILL.md reference': ['browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**'],
|
||||
'setup block': ['SKILL.md', 'SKILL.md.tmpl'],
|
||||
'regression vs baseline': ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json'],
|
||||
'qa/SKILL.md workflow': ['qa/SKILL.md', 'qa/SKILL.md.tmpl'],
|
||||
'qa/SKILL.md health rubric': ['qa/SKILL.md', 'qa/SKILL.md.tmpl'],
|
||||
'qa/SKILL.md anti-refusal': ['qa/SKILL.md', 'qa/SKILL.md.tmpl', 'qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'],
|
||||
'cross-skill greptile consistency': ['review/SKILL.md', 'review/SKILL.md.tmpl', 'ship/SKILL.md', 'ship/SKILL.md.tmpl', 'review/greptile-triage.md', 'retro/SKILL.md', 'retro/SKILL.md.tmpl'],
|
||||
'baseline score pinning': ['SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json'],
|
||||
|
||||
// Ship & Release
|
||||
'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl'],
|
||||
'document-release/SKILL.md workflow': ['document-release/SKILL.md', 'document-release/SKILL.md.tmpl'],
|
||||
|
||||
// Plan Reviews
|
||||
'plan-ceo-review/SKILL.md modes': ['plan-ceo-review/SKILL.md', 'plan-ceo-review/SKILL.md.tmpl'],
|
||||
'plan-eng-review/SKILL.md sections': ['plan-eng-review/SKILL.md', 'plan-eng-review/SKILL.md.tmpl'],
|
||||
|
||||
// /spec authored-spec quality (paid LLM-judge — periodic-tier).
|
||||
'plan-design-review/SKILL.md passes': ['plan-design-review/SKILL.md', 'plan-design-review/SKILL.md.tmpl'],
|
||||
|
||||
// Design skills
|
||||
'design-review/SKILL.md fix loop': ['design-review/SKILL.md', 'design-review/SKILL.md.tmpl'],
|
||||
'design-consultation/SKILL.md research': ['design-consultation/SKILL.md', 'design-consultation/SKILL.md.tmpl'],
|
||||
|
||||
// Office Hours
|
||||
'office-hours/SKILL.md spec review': ['office-hours/SKILL.md', 'office-hours/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'office-hours/SKILL.md design sketch': ['office-hours/SKILL.md', 'office-hours/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Deploy skills
|
||||
'land-and-deploy/SKILL.md workflow': ['land-and-deploy/SKILL.md', 'land-and-deploy/SKILL.md.tmpl'],
|
||||
'canary/SKILL.md monitoring loop': ['canary/SKILL.md', 'canary/SKILL.md.tmpl'],
|
||||
'benchmark/SKILL.md perf collection': ['benchmark/SKILL.md', 'benchmark/SKILL.md.tmpl'],
|
||||
'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl'],
|
||||
|
||||
// Other skills
|
||||
'retro/SKILL.md instructions': ['retro/SKILL.md', 'retro/SKILL.md.tmpl'],
|
||||
'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'],
|
||||
'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl'],
|
||||
|
||||
// Voice directive
|
||||
'voice directive tone': ['scripts/resolvers/preamble.ts', 'review/SKILL.md', 'review/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
};
|
||||
|
||||
/**
|
||||
* Changes to any of these files trigger ALL tests (both E2E and LLM-judge).
|
||||
*
|
||||
* Keep this list minimal — only files that genuinely affect every test.
|
||||
* Scoped dependencies (gen-skill-docs, llm-judge, test-server, worktree,
|
||||
* codex/gemini session runners) belong in individual test entries instead.
|
||||
*/
|
||||
export const GLOBAL_TOUCHFILES = [
|
||||
'test/helpers/session-runner.ts', // All E2E tests use this runner
|
||||
'test/helpers/hermetic-env.ts', // Changes every E2E child's environment
|
||||
'test/helpers/eval-store.ts', // All E2E tests store results here
|
||||
'test/helpers/test-selection.ts', // Selection logic itself — a bug here mis-selects every test
|
||||
'test/helpers/touchfiles.ts', // The facade is executable selection-path code; an edit must run everything (it should never change, so the cost is ~zero)
|
||||
'test/helpers/e2e-helpers.ts', // Shared harness every paid test imports (selection wiring, preflight, describeIfSelected) — an edit here changes every test's behavior
|
||||
'test/helpers/paid-test-set.ts', // Paid-vs-free classification — an edit moves files between suites
|
||||
'test/helpers/skill-fixture.ts', // SKILL.md fixture extraction — reshapes the skill content most E2E suites read
|
||||
// NOTE: this file (touchfiles-data.ts) is deliberately NOT a global
|
||||
// touchfile. Changes to it route through map-diff selection in
|
||||
// test-selection.ts: the old git version is evaluated and the maps are
|
||||
// diffed per key, so a data-only edit runs just the affected tests.
|
||||
// Map-diff fails CLOSED — any error on that path still runs everything.
|
||||
];
|
||||
+35
-867
@@ -1,875 +1,43 @@
|
||||
/**
|
||||
* Diff-based test selection for E2E and LLM-judge evals.
|
||||
* Diff-based test selection for E2E and LLM-judge evals — compatibility facade.
|
||||
*
|
||||
* Each test declares which source files it depends on ("touchfiles").
|
||||
* The test runner checks `git diff` and only runs tests whose
|
||||
* dependencies were modified. Override with EVALS_ALL=1 to run everything.
|
||||
*/
|
||||
|
||||
import { spawnSync } from 'child_process';
|
||||
|
||||
// --- Glob matching ---
|
||||
|
||||
/**
|
||||
* Match a file path against a glob pattern.
|
||||
* Supports:
|
||||
* ** — match any number of path segments
|
||||
* * — match within a single segment (no /)
|
||||
*/
|
||||
export function matchGlob(file: string, pattern: string): boolean {
|
||||
const regexStr = pattern
|
||||
.replace(/\./g, '\\.')
|
||||
.replace(/\*\*/g, '{{GLOBSTAR}}')
|
||||
.replace(/\*/g, '[^/]*')
|
||||
.replace(/\{\{GLOBSTAR\}\}/g, '.*');
|
||||
return new RegExp(`^${regexStr}$`).test(file);
|
||||
}
|
||||
|
||||
// --- Touchfile maps ---
|
||||
|
||||
/**
|
||||
* E2E test touchfiles — keyed by testName (the string passed to runSkillTest).
|
||||
* Each test lists the file patterns that, if changed, require the test to run.
|
||||
*/
|
||||
export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// Browse core (+ test-server dependency)
|
||||
'browse-basic': ['browse/src/**', 'browse/test/test-server.ts'],
|
||||
'browse-snapshot': ['browse/src/**', 'browse/test/test-server.ts'],
|
||||
|
||||
// Hermetic isolation canaries (hermetic-env.ts is also a GLOBAL touchfile;
|
||||
// these entries exist so the canaries themselves stay tier-classified)
|
||||
'hermetic-canary': ['test/helpers/hermetic-env.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-hermetic-canary.test.ts', 'lib/conductor-env-shim.ts'],
|
||||
'hermetic-sentinel': ['test/helpers/hermetic-env.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-hermetic-canary.test.ts', 'lib/conductor-env-shim.ts'],
|
||||
|
||||
// P4 first-run scaffold (activation lift) — the detection binary end-to-end
|
||||
// through the real runner, plus the preamble wiring that gates + maps it.
|
||||
'first-task-scaffold': ['bin/gstack-first-task-detect', 'scripts/resolvers/preamble/generate-first-run-guidance.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'test/skill-e2e-first-task-scaffold.test.ts', 'test/helpers/session-runner.ts'],
|
||||
|
||||
// SKILL.md setup + preamble (depend on ROOT SKILL.md + gen-skill-docs)
|
||||
'skillmd-setup-discovery': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'skillmd-no-local-binary': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'skillmd-outside-git': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
'session-awareness': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'operational-learning': ['scripts/resolvers/preamble.ts', 'bin/gstack-learnings-log'],
|
||||
|
||||
// QA (+ test-server dependency)
|
||||
'qa-quick': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts'],
|
||||
'qa-b6-static': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval.html', 'test/fixtures/qa-eval-ground-truth.json'],
|
||||
'qa-b7-spa': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-spa.html', 'test/fixtures/qa-eval-spa-ground-truth.json'],
|
||||
'qa-b8-checkout': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts', 'test/helpers/llm-judge.ts', 'browse/test/fixtures/qa-eval-checkout.html', 'test/fixtures/qa-eval-checkout-ground-truth.json'],
|
||||
'qa-only-no-fix': ['qa-only/**', 'qa/templates/**'],
|
||||
'qa-fix-loop': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts'],
|
||||
'qa-bootstrap': ['qa/**', 'ship/**'],
|
||||
|
||||
// Review
|
||||
'review-sql-injection': ['review/**', 'test/fixtures/review-eval-vuln.rb'],
|
||||
'review-enum-completeness': ['review/**', 'test/fixtures/review-eval-enum*.rb'],
|
||||
'review-base-branch': ['review/**'],
|
||||
'review-design-lite': ['review/**', 'test/fixtures/review-eval-design-slop.*'],
|
||||
|
||||
// Review Army (specialist dispatch)
|
||||
'review-army-migration-safety': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope'],
|
||||
'review-army-perf-n-plus-one': ['review/**', 'scripts/resolvers/review-army.ts', 'bin/gstack-diff-scope'],
|
||||
'review-army-delivery-audit': ['review/**', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-quality-score': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-json-findings': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-red-team': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
'review-army-consensus': ['review/**', 'scripts/resolvers/review-army.ts'],
|
||||
|
||||
// Office Hours
|
||||
'office-hours-spec-review': ['office-hours/**', 'scripts/gen-skill-docs.ts'],
|
||||
'office-hours-forcing-energy': ['office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts'],
|
||||
'office-hours-builder-wildness': ['office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts'],
|
||||
|
||||
// Plan reviews
|
||||
'plan-ceo-review': ['plan-ceo-review/**'],
|
||||
'plan-ceo-review-selective': ['plan-ceo-review/**'],
|
||||
'plan-ceo-review-benefits': ['plan-ceo-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
'plan-ceo-review-expansion-energy': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts'],
|
||||
'plan-eng-review': ['plan-eng-review/**'],
|
||||
'plan-eng-review-artifact': ['plan-eng-review/**'],
|
||||
'plan-review-report': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Plan-mode smoke tests — gate-tier safety regression tests. Each test file
|
||||
// contains TWO test cases as of v1.21: the baseline plan-mode case and the
|
||||
// AskUserQuestion-blocked regression case (--disallowedTools AskUserQuestion
|
||||
// parameterized — the flag set Conductor uses by default). Touchfiles
|
||||
// include question-tuning.ts and generate-ask-user-format.ts because the
|
||||
// AUTO_DECIDE preamble injection lives there and changes can flip the
|
||||
// regression test outcome between 'asked' and 'auto_decided'.
|
||||
'plan-ceo-review-plan-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'],
|
||||
'plan-eng-review-plan-mode': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'],
|
||||
'plan-design-review-plan-mode': ['plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts'],
|
||||
'plan-devex-review-plan-mode': ['plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
// Covers ceo (preamble misfire) + eng/design (scope-gate bypass must not
|
||||
// fire outside plan mode) + the named-target exception case. 4 PTY runs;
|
||||
// in CI these run CONCURRENT with the rest of the pty-plan-smoke suite
|
||||
// (--max-concurrency + --retry 2), so worst-case cost is ~3x a single
|
||||
// pass of each, sharing the API budget with sibling tests — not the
|
||||
// sequential ~+10min a local read suggests.
|
||||
'plan-mode-no-op': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-mode-no-op.test.ts'],
|
||||
|
||||
// v1.21+ AskUserQuestion-blocked regression tests — Conductor launches
|
||||
// claude with `--disallowedTools AskUserQuestion --permission-mode default`
|
||||
// (verified via `ps`); skills must still surface user-decisions through a
|
||||
// fallback path (mcp__conductor__AskUserQuestion or plan-file flow) rather
|
||||
// than silently auto-deciding. Parameterized regression test cases live
|
||||
// INSIDE the existing 4 plan-X-review-plan-mode test files (covered
|
||||
// transitively by the entries above). Two new standalone files exist for
|
||||
// skills with no prior plan-mode test:
|
||||
'office-hours-auto-mode': ['office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
'office-hours-phase4-fork': ['office-hours/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/question-tuning.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours-phase4.test.ts'],
|
||||
'llm-judge-recommendation': ['test/helpers/llm-judge.ts', 'test/llm-judge-recommendation.test.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'codex/SKILL.md.tmpl', 'scripts/resolvers/review.ts'],
|
||||
// v1.21+ AUTO_DECIDE preserve eval (periodic). Verifies the Tool resolution
|
||||
// fix doesn't trip the legitimate /plan-tune opt-in path: when the user has
|
||||
// written a never-ask preference, AUQ should still auto-decide rather than
|
||||
// surfacing the question. Touches the question-tuning + preference
|
||||
// infrastructure plus the resolvers that own the AUTO_DECIDE preamble.
|
||||
'auto-decide-preserved': ['scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
|
||||
// Conductor → prose decision brief (Conductor signal makes prose the default;
|
||||
// the PreToolUse hook denies the flaky tool). Touches the resolver that owns
|
||||
// the Conductor rule, the preamble signal, the hook, and the detection helper.
|
||||
'conductor-prose': ['scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble.ts', 'plan-eng-review/**', 'hosts/claude/hooks/question-preference-hook.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-conductor-prose.test.ts'],
|
||||
|
||||
// Real-PTY E2E batch (#6 new tests on the harness).
|
||||
// Each one tests behavior the SDK harness can't observe (rendered TTY,
|
||||
// numbered-option lists, multi-phase ordering, idempotency state echo).
|
||||
'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts'],
|
||||
'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts'],
|
||||
'budget-regression-pty': ['test/helpers/eval-store.ts', 'test/skill-budget-regression.test.ts'],
|
||||
'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
'ship-section-loading': ['ship/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
|
||||
'plan-ceo-section-loading': ['plan-ceo-review/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
|
||||
// Data-driven behavioral guard for the 'plan'/'prompt' carves (eng, design,
|
||||
// devex, office-hours + future PR2 carves). One file iterating CARVE_GUARDS;
|
||||
// the selector sets GSTACK_CARVE_SKILL=<name> to scope cost to the changed
|
||||
// skill (D-CODEX A). Touching the registry/helper or sections.ts runs all.
|
||||
'carve-section-loading': ['plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
|
||||
'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts'],
|
||||
'e2e-harness-audit': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
|
||||
// Per-finding AskUserQuestion count + review-report-at-bottom assertion.
|
||||
// Each test drives its skill end-to-end; touchfiles include preamble +
|
||||
// completion-status resolvers because they affect question cadence and
|
||||
// terminal output (the regression surface this test catches).
|
||||
'plan-ceo-finding-count': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'],
|
||||
'plan-eng-finding-count': ['plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'],
|
||||
'plan-design-finding-count': ['plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'],
|
||||
'plan-devex-finding-count': ['plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'],
|
||||
|
||||
// Gate-tier reviewCount-floor counterparts. Catch the May 2026 transcript
|
||||
// bug (model wrote a plan-mode plan and ExitPlanMode'd without firing any
|
||||
// review-phase AskUserQuestion). Uses runPlanSkillFloorCheck — minimal
|
||||
// "did agent fire ANY AUQ?" observer that exits early on first non-permission
|
||||
// numbered-option render. ~1-3 min typical wall time per test, ~$2-6 total.
|
||||
'plan-eng-finding-floor': ['plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-finding-floor.test.ts'],
|
||||
'plan-ceo-finding-floor': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-finding-floor.test.ts'],
|
||||
'plan-design-finding-floor': ['plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-design-finding-floor.test.ts'],
|
||||
'plan-devex-finding-floor': ['plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-devex-finding-floor.test.ts'],
|
||||
|
||||
// Multi-finding batching regression — periodic tier complement to the
|
||||
// gate-tier finding-floor. Catches the May 2026 transcript shape where
|
||||
// a model fires one AUQ then batches the rest into a "## Decisions to
|
||||
// confirm" plan write. runPlanSkillFloorCheck cannot detect that shape
|
||||
// (it exits on first AUQ); runPlanSkillCounting can.
|
||||
'plan-eng-multi-finding-batching': ['plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'],
|
||||
'plan-ceo-split-overflow': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'],
|
||||
'brain-privacy-gate': ['scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts'],
|
||||
|
||||
// /setup-gbrain Path 4 (Remote MCP) — happy + bad-token end-to-end via
|
||||
// Agent SDK. Gate-tier (deterministic stub server, fixed inputs); fires
|
||||
// when the skill template, the verify helper, the artifacts-init helper,
|
||||
// or the detect script changes.
|
||||
'setup-gbrain-remote': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'bin/gstack-artifacts-init', 'bin/gstack-gbrain-detect', 'test/helpers/agent-sdk-runner.ts'],
|
||||
'setup-gbrain-bad-token': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'test/helpers/agent-sdk-runner.ts'],
|
||||
// v1.34.0.0 split-engine Path 4 + Step 4.5 Yes (local PGLite for code).
|
||||
// Periodic-tier per codex #12 (AgentSDK harness is non-deterministic).
|
||||
// Fires when the setup-gbrain template, install/verify/init helpers, or
|
||||
// the agent-sdk-runner harness changes.
|
||||
'setup-gbrain-path4-local-pglite': ['setup-gbrain/SKILL.md.tmpl', 'bin/gstack-gbrain-mcp-verify', 'bin/gstack-gbrain-install', 'bin/gstack-gbrain-detect', 'lib/gbrain-local-status.ts', 'test/helpers/agent-sdk-runner.ts'],
|
||||
|
||||
// AskUserQuestion format regression (RECOMMENDATION + Completeness: N/10)
|
||||
// Fires when either template OR the two preamble resolvers change.
|
||||
'plan-ceo-review-format-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'],
|
||||
'plan-ceo-review-format-approach': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'],
|
||||
'plan-eng-review-format-coverage': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'],
|
||||
'plan-eng-review-format-kind': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md', 'test/helpers/llm-judge.ts'],
|
||||
|
||||
// v1.7.0.0 Pros/Cons format cadence + format + negative-escape evals.
|
||||
// Dependencies: same as format-mode + the 4 plan-review templates + overlay.
|
||||
// All periodic-tier (non-deterministic Opus 4.7 behavior).
|
||||
'plan-ceo-review-prosons-cadence': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'plan-review-prosons-format': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'plan-review-prosons-hardstop-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'plan-review-prosons-neutral-neg': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
|
||||
// Expanded coverage (CT3) — 6 non-plan-review skills inherit Pros/Cons via preamble
|
||||
'ship-prosons-format': ['ship/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'office-hours-prosons-format': ['office-hours/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'investigate-prosons-format': ['investigate/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'qa-prosons-format': ['qa/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'review-prosons-format': ['review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'design-review-prosons-format': ['design-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
'document-release-prosons-format': ['document-release/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'model-overlays/opus-4-7.md'],
|
||||
|
||||
// /plan-tune (v1 observational)
|
||||
'plan-tune-inspect': ['plan-tune/**', 'scripts/question-registry.ts', 'scripts/psychographic-signals.ts', 'scripts/one-way-doors.ts', 'bin/gstack-question-log', 'bin/gstack-question-preference', 'bin/gstack-developer-profile'],
|
||||
|
||||
// /plan-tune cathedral (T16 — 5 E2E scenarios, all gate per D12)
|
||||
'plan-tune-hook-capture': ['hosts/claude/hooks/**', 'bin/gstack-question-log', 'bin/gstack-developer-profile', 'plan-tune/**'],
|
||||
'plan-tune-enforcement': ['hosts/claude/hooks/**', 'bin/gstack-question-preference', 'scripts/question-registry.ts'],
|
||||
'plan-tune-annotation': ['hosts/claude/hooks/**', 'scripts/declared-annotation.ts', 'scripts/psychographic-signals.ts', 'scripts/question-registry.ts'],
|
||||
'plan-tune-codex-import': ['bin/gstack-codex-session-import', 'bin/gstack-question-log', 'docs/spikes/codex-session-format.md'],
|
||||
'plan-tune-dream-cycle': ['bin/gstack-distill-free-text', 'bin/gstack-distill-apply', 'hosts/claude/hooks/**', 'plan-tune/**'],
|
||||
|
||||
// Codex offering verification
|
||||
'codex-offered-office-hours': ['office-hours/**', 'scripts/gen-skill-docs.ts'],
|
||||
'codex-offered-ceo-review': ['plan-ceo-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
'codex-offered-design-review': ['plan-design-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
'codex-offered-eng-review': ['plan-eng-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Ship
|
||||
'ship-base-branch': ['ship/**', 'bin/gstack-repo-mode'],
|
||||
'ship-local-workflow': ['ship/**', 'scripts/gen-skill-docs.ts'],
|
||||
'review-dashboard-via': ['ship/**', 'scripts/resolvers/review.ts', 'codex/**', 'autoplan/**', 'land-and-deploy/**'],
|
||||
'ship-plan-completion': ['ship/**', 'scripts/gen-skill-docs.ts'],
|
||||
'ship-plan-verification': ['ship/**', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Retro
|
||||
'retro': ['retro/**'],
|
||||
'retro-base-branch': ['retro/**'],
|
||||
|
||||
// Global discover
|
||||
'global-discover': ['bin/gstack-global-discover.ts', 'test/global-discover.test.ts'],
|
||||
|
||||
// CSO
|
||||
'cso-full-audit': ['cso/**'],
|
||||
'cso-diff-mode': ['cso/**'],
|
||||
'cso-infra-scope': ['cso/**'],
|
||||
|
||||
// Learnings
|
||||
'learnings-show': ['learn/**', 'bin/gstack-learnings-search', 'bin/gstack-learnings-log', 'scripts/resolvers/learnings.ts'],
|
||||
|
||||
// Session Intelligence (timeline, context recovery, /context-save + /context-restore)
|
||||
'timeline-event-flow': ['bin/gstack-timeline-log', 'bin/gstack-timeline-read'],
|
||||
'context-recovery-artifacts': ['scripts/resolvers/preamble.ts', 'bin/gstack-timeline-log', 'bin/gstack-slug', 'learn/**'],
|
||||
'context-save-writes-file': ['context-save/**', 'bin/gstack-slug'],
|
||||
'context-restore-loads-latest': ['context-restore/**', 'bin/gstack-slug'],
|
||||
|
||||
// Context skills E2E (live-fire, Skill-tool routing path) — see
|
||||
// test/skill-e2e-context-skills.test.ts. These are periodic-tier because
|
||||
// each one spawns claude -p and costs ~$0.20-$0.40. Collectively they
|
||||
// verify the thing the /checkpoint → /context-save rename was for.
|
||||
'context-save-routing': ['context-save/**', 'scripts/resolvers/preamble.ts'],
|
||||
'context-save-then-restore-roundtrip': ['context-save/**', 'context-restore/**', 'bin/gstack-slug'],
|
||||
'context-restore-fragment-match': ['context-restore/**'],
|
||||
'context-restore-empty-state': ['context-restore/**'],
|
||||
'context-restore-list-delegates': ['context-restore/**'],
|
||||
'context-restore-legacy-compat': ['context-restore/**'],
|
||||
'context-save-list-current-branch': ['context-save/**'],
|
||||
'context-save-list-all-branches': ['context-save/**'],
|
||||
|
||||
// Document-release
|
||||
'document-release': ['document-release/**'],
|
||||
|
||||
// Codex (Claude E2E — tests /codex skill via Claude)
|
||||
'codex-review': ['codex/**'],
|
||||
|
||||
// Codex E2E (tests skills via Codex CLI + worktree)
|
||||
'codex-discover-skill': ['codex/**', '.agents/skills/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts'],
|
||||
'codex-review-findings': ['review/**', '.agents/skills/gstack-review/**', 'codex/**', 'test/helpers/codex-session-runner.ts', 'lib/worktree.ts'],
|
||||
|
||||
// Gemini E2E — smoke test only (Gemini gets lost in worktrees on complex tasks)
|
||||
'gemini-smoke': ['.agents/skills/**', 'test/helpers/gemini-session-runner.ts', 'lib/worktree.ts'],
|
||||
|
||||
|
||||
// Coverage audit (shared fixture) + triage + gates
|
||||
'ship-coverage-audit': ['ship/**', 'test/fixtures/coverage-audit-fixture.ts', 'bin/gstack-repo-mode'],
|
||||
'review-coverage-audit': ['review/**', 'test/fixtures/coverage-audit-fixture.ts'],
|
||||
'plan-eng-coverage-audit': ['plan-eng-review/**', 'test/fixtures/coverage-audit-fixture.ts'],
|
||||
'ship-triage': ['ship/**', 'bin/gstack-repo-mode'],
|
||||
|
||||
// Plan completion audit + verification
|
||||
'ship-plan-completion': ['ship/**', 'scripts/gen-skill-docs.ts'],
|
||||
'ship-plan-verification': ['ship/**', 'qa-only/**', 'scripts/gen-skill-docs.ts'],
|
||||
'review-plan-completion': ['review/**', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Design
|
||||
'design-consultation-core': ['design-consultation/**', 'scripts/gen-skill-docs.ts', 'test/helpers/llm-judge.ts'],
|
||||
'design-consultation-existing': ['design-consultation/**', 'scripts/gen-skill-docs.ts'],
|
||||
'design-consultation-research': ['design-consultation/**', 'scripts/gen-skill-docs.ts'],
|
||||
'design-consultation-preview': ['design-consultation/**', 'scripts/gen-skill-docs.ts'],
|
||||
'plan-design-review-no-ui-scope': ['plan-design-review/**', 'scripts/gen-skill-docs.ts'],
|
||||
'design-review-fix': ['design-review/**', 'browse/src/**', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Design Shotgun
|
||||
'design-shotgun-path': ['design-shotgun/**', 'design/src/**', 'scripts/resolvers/design.ts'],
|
||||
'design-shotgun-session': ['design-shotgun/**', 'scripts/resolvers/design.ts'],
|
||||
'design-shotgun-full': ['design-shotgun/**', 'design/src/**', 'browse/src/**'],
|
||||
|
||||
// /diagram (diagram-render bundle consumers). Triplet = deterministic
|
||||
// functional (gate); authoring quality = LLM-judged benchmark (periodic).
|
||||
'diagram-triplet': ['diagram/**', 'lib/diagram-render/**', 'browse/src/write-commands.ts', 'browse/src/read-commands.ts'],
|
||||
'diagram-authoring-quality': ['diagram/**', 'lib/diagram-render/**', 'test/helpers/llm-judge.ts'],
|
||||
|
||||
// gstack-upgrade
|
||||
'gstack-upgrade-happy-path': ['gstack-upgrade/**'],
|
||||
|
||||
// Deploy skills
|
||||
'land-and-deploy-workflow': ['land-and-deploy/**', 'scripts/gen-skill-docs.ts'],
|
||||
'land-and-deploy-first-run': ['land-and-deploy/**', 'scripts/gen-skill-docs.ts', 'bin/gstack-slug'],
|
||||
'land-and-deploy-review-gate': ['land-and-deploy/**', 'bin/gstack-review-read'],
|
||||
'canary-workflow': ['canary/**', 'browse/src/**'],
|
||||
'benchmark-workflow': ['benchmark/**', 'browse/src/**'],
|
||||
'setup-deploy-workflow': ['setup-deploy/**', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Autoplan
|
||||
'autoplan-core': ['autoplan/**', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**'],
|
||||
'autoplan-dual-voice': ['autoplan/**', 'codex/**', 'bin/gstack-codex-probe', 'scripts/resolvers/review.ts', 'scripts/resolvers/design.ts'],
|
||||
|
||||
// Multi-provider benchmark adapters — live API smoke against real claude/codex/gemini CLIs
|
||||
'benchmark-providers-live': ['bin/gstack-model-benchmark', 'test/helpers/providers/**', 'test/helpers/benchmark-runner.ts', 'test/helpers/pricing.ts'],
|
||||
|
||||
// Browser-skills Phase 2a — /scrape + /skillify (v1.19.0.0). Gate-tier
|
||||
// E2E covers the D1 (provenance guard), D3 (atomic write) contracts plus
|
||||
// the basic loop. Shared deps: both skill templates, the D3 helper, the
|
||||
// Phase 1 runtime, and the bundled hackernews-frontpage reference (the
|
||||
// match-path test relies on it).
|
||||
'scrape-match-path': [
|
||||
'scrape/**', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts',
|
||||
'browser-skills/hackernews-frontpage/**',
|
||||
],
|
||||
'scrape-prototype-path': [
|
||||
'scrape/**', 'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts',
|
||||
],
|
||||
'skillify-happy-path': [
|
||||
'skillify/**', 'scrape/**', 'browse/src/browser-skill-write.ts',
|
||||
'browse/src/browser-skills.ts', 'browse/src/browser-skill-commands.ts',
|
||||
],
|
||||
'skillify-provenance-refusal': [
|
||||
'skillify/**', 'browse/src/browser-skill-write.ts',
|
||||
],
|
||||
'skillify-approval-reject': [
|
||||
'skillify/**', 'scrape/**', 'browse/src/browser-skill-write.ts',
|
||||
],
|
||||
|
||||
// Skill routing — journey-stage tests (depend on ALL skill descriptions)
|
||||
'journey-ideation': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-plan-eng': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-debug': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-code-review': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-ship': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-docs': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-retro': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-design-system': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'journey-visual-qa': ['*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Opus 4.7 behavior evals — keys match testName: values in the test file.
|
||||
// Routing sub-tests use template literal `routing-${c.name}` testNames,
|
||||
// which the touchfile completeness scanner skips; they inherit selection
|
||||
// from the file-level touchfile entry via GLOBAL_TOUCHFILES.
|
||||
'fanout-arm-overlay-on':
|
||||
['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts'],
|
||||
'fanout-arm-overlay-off':
|
||||
['model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts'],
|
||||
|
||||
// Overlay efficacy harness (SDK) — measures whether overlay nudges change
|
||||
// behavior under @anthropic-ai/claude-agent-sdk (closer to real Claude Code
|
||||
// than `claude -p`). testNames in the file are template literals so the
|
||||
// completeness scanner doesn't require them; these entries exist for
|
||||
// diff-based selection accuracy.
|
||||
'overlay-harness-opus-4-7-fanout-toy': [
|
||||
'model-overlays/**',
|
||||
'test/fixtures/overlay-nudges.ts',
|
||||
'test/helpers/agent-sdk-runner.ts',
|
||||
'scripts/resolvers/model-overlay.ts',
|
||||
],
|
||||
'overlay-harness-opus-4-7-fanout-realistic': [
|
||||
'model-overlays/**',
|
||||
'test/fixtures/overlay-nudges.ts',
|
||||
'test/helpers/agent-sdk-runner.ts',
|
||||
'scripts/resolvers/model-overlay.ts',
|
||||
],
|
||||
|
||||
// /ios-qa — agent flow E2E. Daemon + stub StateServer + codegen
|
||||
// exercised end-to-end. The no-device path is gate-tier; the with-device
|
||||
// path requires GSTACK_HAS_IOS_DEVICE=1 and is periodic-tier.
|
||||
'ios-qa-e2e': ['ios-qa/**', 'ios-fix/**', 'ios-design-review/**', 'ios-clean/**', 'ios-sync/**', 'test/skill-e2e-ios.test.ts'],
|
||||
// Swift-build invariant test — requires the Swift toolchain. Compiles the
|
||||
// fixture SPM package + runs the XCTest suite that validates the real
|
||||
// Swift StateServer implementation (loopback bind, boot token rotation,
|
||||
// session lock). Periodic-tier — Swift build is heavier than TS unit tests.
|
||||
'ios-qa-swift-build': ['ios-qa/templates/**', 'test/fixtures/ios-qa/FixtureApp/**', 'test/skill-e2e-ios-swift-build.test.ts'],
|
||||
// Real-device path — only runs with GSTACK_HAS_IOS_DEVICE=1 + a paired
|
||||
// iPhone. Validates the CoreDevice agent + iOS SDK toolchain. Periodic-tier.
|
||||
'ios-qa-device': ['ios-qa/templates/**', 'test/fixtures/ios-qa/FixtureApp/**', 'test/skill-e2e-ios-device.test.ts'],
|
||||
|
||||
// /spec end-to-end via PTY — exercises the full Phase 1→5 pipeline
|
||||
// including --execute spawn. Periodic-tier — paid + non-deterministic.
|
||||
'spec-execute': ['spec/**', 'test/skill-e2e-spec-execute.test.ts'],
|
||||
|
||||
// /office-hours brain-writeback path under fake gbrain CLI (v1.50.0.0
|
||||
// T7). Drives /office-hours with a regenerated SKILL.md that has the
|
||||
// compressed GBRAIN_SAVE_RESULTS block + a fake gbrain on PATH; asserts
|
||||
// the agent calls `gbrain put office-hours/<slug>` with valid YAML
|
||||
// frontmatter. Touched by anything that changes resolver output, gen
|
||||
// pipeline, detection helper, refresh subcommand, or the on-demand
|
||||
// docs the resolver points to.
|
||||
'office-hours-brain-writeback': [
|
||||
'scripts/resolvers/gbrain.ts',
|
||||
'scripts/gen-skill-docs.ts',
|
||||
'bin/gstack-gbrain-detect',
|
||||
'bin/gstack-config',
|
||||
'office-hours/SKILL.md.tmpl',
|
||||
'docs/gbrain-write-surfaces.md',
|
||||
'test/fixtures/office-hours-brain-writeback/**',
|
||||
'test/skill-e2e-office-hours-brain-writeback.test.ts',
|
||||
],
|
||||
|
||||
// gbrain CLI real round-trip against a local PGLite store (v1.50.0.0
|
||||
// T11). Proves the gbrain CLI persistence contract gstack relies on —
|
||||
// a `gbrain put` followed by `gbrain get` returns the body. Skips if
|
||||
// VOYAGE_API_KEY is unset OR gbrain CLI not on PATH. Touched by the
|
||||
// resolver (which emits the CLI shape) and the test itself.
|
||||
'gbrain-roundtrip-local': [
|
||||
'scripts/resolvers/gbrain.ts',
|
||||
'test/skill-e2e-gbrain-roundtrip-local.test.ts',
|
||||
],
|
||||
|
||||
};
|
||||
|
||||
/**
|
||||
* E2E test tiers — 'gate' blocks PRs, 'periodic' runs weekly/on-demand.
|
||||
* Must have exactly the same keys as E2E_TOUCHFILES.
|
||||
*/
|
||||
export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
// Browse core — gate (if browse breaks, everything breaks)
|
||||
'browse-basic': 'gate',
|
||||
'browse-snapshot': 'gate',
|
||||
|
||||
// Hermetic isolation — gate (deterministic env/config assertions; if the
|
||||
// clean room breaks, every other eval's signal is contaminated)
|
||||
'hermetic-canary': 'gate',
|
||||
'hermetic-sentinel': 'gate',
|
||||
|
||||
// SKILL.md setup — gate (if setup breaks, no skill works)
|
||||
'skillmd-setup-discovery': 'gate',
|
||||
'skillmd-no-local-binary': 'gate',
|
||||
'skillmd-outside-git': 'gate',
|
||||
'session-awareness': 'gate',
|
||||
'operational-learning': 'gate',
|
||||
|
||||
// P4 first-run scaffold — periodic (onboarding, non-safety, model-touched marker)
|
||||
'first-task-scaffold': 'periodic',
|
||||
|
||||
// QA — gate for functional, periodic for quality/benchmarks
|
||||
'qa-quick': 'gate',
|
||||
'qa-b6-static': 'periodic',
|
||||
'qa-b7-spa': 'periodic',
|
||||
'qa-b8-checkout': 'periodic',
|
||||
'qa-only-no-fix': 'gate', // CRITICAL guardrail: Edit tool forbidden
|
||||
'qa-fix-loop': 'periodic',
|
||||
'qa-bootstrap': 'gate',
|
||||
|
||||
// Review — gate for functional/guardrails, periodic for quality
|
||||
'review-sql-injection': 'gate', // Security guardrail
|
||||
'review-enum-completeness': 'gate',
|
||||
'review-base-branch': 'gate',
|
||||
'review-design-lite': 'periodic', // 4/7 threshold is subjective
|
||||
'review-coverage-audit': 'gate',
|
||||
'review-plan-completion': 'gate',
|
||||
'review-dashboard-via': 'gate',
|
||||
|
||||
// Review Army — gate for core functionality, periodic for multi-specialist
|
||||
'review-army-migration-safety': 'gate', // Specialist activation guardrail
|
||||
'review-army-perf-n-plus-one': 'gate', // Specialist activation guardrail
|
||||
'review-army-delivery-audit': 'gate', // Delivery integrity guardrail
|
||||
'review-army-quality-score': 'gate', // Score computation
|
||||
'review-army-json-findings': 'gate', // JSON schema compliance
|
||||
'review-army-red-team': 'periodic', // Multi-agent coordination
|
||||
'review-army-consensus': 'periodic', // Multi-specialist agreement
|
||||
|
||||
// Office Hours
|
||||
'office-hours-spec-review': 'gate',
|
||||
// Brain-writeback E2E — periodic per cost (claude -p) + non-deterministic
|
||||
// (model interprets the gbrain instruction). Matches nearby
|
||||
// setup-gbrain-path4-* tier classification.
|
||||
'office-hours-brain-writeback': 'periodic',
|
||||
// GBrain CLI round-trip — periodic per Voyage embedding cost (~$0.001/run)
|
||||
// and external-API-dependency (skips cleanly if VOYAGE_API_KEY unset).
|
||||
'gbrain-roundtrip-local': 'periodic',
|
||||
'office-hours-forcing-energy': 'gate', // V1.1 mode-posture regression gate (Sonnet generator)
|
||||
// 'office-hours-builder-wildness' retiered to periodic in v1.32 contributor
|
||||
// wave: this is an LLM-judge creativity score (axis_a ≥4 on a "wildness"
|
||||
// posture). Per CLAUDE.md tier-classification rules, non-deterministic
|
||||
// quality benchmarks belong in periodic, not gate. The wave's +21-line
|
||||
// CJK preamble cascade (#1205) pushed the score from 5/5 → 3/3 on the
|
||||
// same /office-hours BUILDER prompt — same model, same fixture — proving
|
||||
// the bar is sensitive to preamble-byte changes that have nothing to do
|
||||
// with the test's intent (creativity, not preamble compliance).
|
||||
'office-hours-builder-wildness': 'periodic',
|
||||
|
||||
// Plan reviews — gate for cheap functional, periodic for Opus quality
|
||||
'plan-ceo-review': 'periodic',
|
||||
'plan-ceo-review-selective': 'periodic',
|
||||
'plan-ceo-review-benefits': 'gate',
|
||||
'plan-ceo-review-expansion-energy': 'gate', // V1.1 mode-posture regression gate (Opus generator, Sonnet judge)
|
||||
'plan-eng-review': 'periodic',
|
||||
'plan-eng-review-artifact': 'periodic',
|
||||
'plan-eng-coverage-audit': 'gate',
|
||||
'plan-review-report': 'gate',
|
||||
|
||||
// Plan-mode handshake. plan-ceo/plan-devex ask-first reliably (gate-tier);
|
||||
// plan-eng/plan-design run a long explore/audit before their first
|
||||
// AskUserQuestion, so whether they reach a terminal outcome within the 300s
|
||||
// budget hinges on stochastic ask-first compliance (~50-67%/run measured).
|
||||
// Per the "non-deterministic -> periodic" tiering rule they are periodic:
|
||||
// the hardened ask-first gate + the collapsed-form detector lifted them from
|
||||
// always-failing to mostly-passing, but they are not deterministic gates.
|
||||
'plan-ceo-review-plan-mode': 'gate',
|
||||
'plan-eng-review-plan-mode': 'periodic',
|
||||
'plan-design-review-plan-mode': 'periodic',
|
||||
'plan-devex-review-plan-mode': 'gate',
|
||||
'plan-mode-no-op': 'gate',
|
||||
// v1.21+ auto-mode regression tests
|
||||
'office-hours-auto-mode': 'gate',
|
||||
'auto-decide-preserved': 'periodic',
|
||||
'conductor-prose': 'periodic',
|
||||
'e2e-harness-audit': 'gate',
|
||||
|
||||
// Real-PTY E2E batch — tier classification:
|
||||
// gate: cheap, deterministic, run on every PR
|
||||
// periodic: long-running or expensive (>$3/run), run weekly
|
||||
'auq-format-gate': 'gate', // ~$0.50/run, SDK capture, single skill probe
|
||||
'plan-ceo-mode-routing': 'periodic', // ~$3/run, deep navigation through 8-12 prior AskUserQuestions
|
||||
'plan-design-with-ui-scope': 'gate', // ~$0.80/run
|
||||
'budget-regression-pty': 'gate', // free, library-only assertion
|
||||
'ship-idempotency-pty': 'periodic', // ~$3/run, real /ship in plan mode
|
||||
'ship-section-loading': 'periodic', // ~$3/run, real /ship; asserts section reads
|
||||
'plan-ceo-section-loading': 'periodic', // ~$3-5/run, real /plan-ceo-review; asserts section read
|
||||
'carve-section-loading': 'periodic', // ~$1-2/skill, data-driven; GSTACK_CARVE_SKILL scopes to one
|
||||
'autoplan-chain-pty': 'periodic', // ~$8/run, all 3 phases sequential
|
||||
|
||||
// Per-finding count + review-report-at-bottom — periodic because each
|
||||
// run drives a full skill end-to-end (~25 min, ~$5/run). Sequential
|
||||
// execution during calibration; concurrent opt-in only after measured
|
||||
// comparison agrees (plan §D15).
|
||||
'plan-ceo-finding-count': 'periodic',
|
||||
'plan-eng-finding-count': 'periodic',
|
||||
'plan-design-finding-count': 'periodic',
|
||||
'plan-devex-finding-count': 'periodic',
|
||||
'plan-eng-finding-floor': 'periodic', // stochastic ask-first (see plan-mode-handshake note); periodic
|
||||
'plan-ceo-finding-floor': 'gate',
|
||||
'plan-design-finding-floor': 'periodic', // stochastic ask-first (see plan-mode-handshake note); periodic
|
||||
'plan-devex-finding-floor': 'gate',
|
||||
'plan-eng-multi-finding-batching': 'periodic',
|
||||
'plan-ceo-split-overflow': 'periodic',
|
||||
|
||||
// Privacy gate for gstack-brain-sync — periodic (non-deterministic LLM call,
|
||||
// costs ~$0.30-$0.50 per run, not needed on every commit)
|
||||
'brain-privacy-gate': 'periodic',
|
||||
|
||||
// /setup-gbrain Path 4 (Remote MCP) — periodic-tier. The stub HTTP
|
||||
// server is deterministic but the model's interpretation of "follow
|
||||
// Path 4 only" is not — assertions on which steps the model ran are
|
||||
// flaky. The deterministic gate-tier coverage for Path 4 lives in
|
||||
// test/setup-gbrain-path4-structure.test.ts (free, <200ms). These
|
||||
// E2E tests stay available for on-demand verification of the live
|
||||
// model's behavior against a stub MCP server.
|
||||
'setup-gbrain-remote': 'periodic',
|
||||
'setup-gbrain-bad-token': 'periodic',
|
||||
'setup-gbrain-path4-local-pglite': 'periodic',
|
||||
|
||||
// AskUserQuestion format regression — periodic (Opus 4.7 non-deterministic benchmark)
|
||||
'plan-ceo-review-format-mode': 'periodic',
|
||||
'plan-ceo-review-format-approach': 'periodic',
|
||||
'plan-eng-review-format-coverage': 'periodic',
|
||||
'plan-eng-review-format-kind': 'periodic',
|
||||
|
||||
// Office-hours Phase 4 silent-auto-decide regression — periodic (Phase 4
|
||||
// requires the agent to invent 2-3 architectures, more open-ended than the
|
||||
// 4 plan-format cases above). Reclassify to gate if it turns out stable.
|
||||
'office-hours-phase4-fork': 'periodic',
|
||||
// judgeRecommendation rubric sanity (fixture-based, ~$0.04/run via Haiku)
|
||||
'llm-judge-recommendation': 'periodic',
|
||||
|
||||
// v1.7.0.0 Pros/Cons format — cadence + negative-escape evals (all periodic)
|
||||
'plan-ceo-review-prosons-cadence': 'periodic',
|
||||
'plan-review-prosons-format': 'periodic',
|
||||
'plan-review-prosons-hardstop-neg': 'periodic',
|
||||
'plan-review-prosons-neutral-neg': 'periodic',
|
||||
|
||||
// CT3 expanded coverage — non-plan-review skills inheriting Pros/Cons (all periodic)
|
||||
'ship-prosons-format': 'periodic',
|
||||
'office-hours-prosons-format': 'periodic',
|
||||
'investigate-prosons-format': 'periodic',
|
||||
'qa-prosons-format': 'periodic',
|
||||
'review-prosons-format': 'periodic',
|
||||
'design-review-prosons-format': 'periodic',
|
||||
'document-release-prosons-format': 'periodic',
|
||||
|
||||
// /plan-tune — gate (core v1 DX promise: plain-English intent routing)
|
||||
'plan-tune-inspect': 'gate',
|
||||
|
||||
// /plan-tune cathedral (T16 per D12 — all gate)
|
||||
'plan-tune-hook-capture': 'gate',
|
||||
'plan-tune-enforcement': 'gate',
|
||||
'plan-tune-annotation': 'gate',
|
||||
'plan-tune-codex-import': 'gate',
|
||||
'plan-tune-dream-cycle': 'gate',
|
||||
|
||||
// Codex offering verification
|
||||
'codex-offered-office-hours': 'gate',
|
||||
'codex-offered-ceo-review': 'gate',
|
||||
'codex-offered-design-review': 'gate',
|
||||
'codex-offered-eng-review': 'gate',
|
||||
|
||||
// Session Intelligence — gate for data flow, periodic for agent integration
|
||||
'timeline-event-flow': 'gate', // Binary data flow (no LLM needed)
|
||||
'context-recovery-artifacts': 'gate', // Preamble reads seeded artifacts
|
||||
'context-save-writes-file': 'gate', // /context-save writes a file
|
||||
'context-restore-loads-latest': 'gate', // Cross-branch newest-by-filename restore
|
||||
|
||||
// Context skills live-fire — periodic (each test spawns claude -p, ~$0.20-$0.40)
|
||||
'context-save-routing': 'periodic', // Proves /context-save routes via Skill tool
|
||||
'context-save-then-restore-roundtrip': 'periodic', // Full cycle in one session
|
||||
'context-restore-fragment-match': 'periodic', // /context-restore <fragment>
|
||||
'context-restore-empty-state': 'periodic', // Graceful zero-saves message
|
||||
'context-restore-list-delegates': 'periodic', // /context-restore list redirect
|
||||
'context-restore-legacy-compat': 'periodic', // Pre-rename files still load
|
||||
'context-save-list-current-branch': 'periodic', // Default branch filter
|
||||
'context-save-list-all-branches': 'periodic', // --all flag
|
||||
|
||||
// Ship — gate (end-to-end ship path)
|
||||
'ship-base-branch': 'gate',
|
||||
'ship-local-workflow': 'gate',
|
||||
'ship-coverage-audit': 'gate',
|
||||
'ship-triage': 'gate',
|
||||
'ship-plan-completion': 'gate',
|
||||
'ship-plan-verification': 'gate',
|
||||
|
||||
// Retro — gate for cheap branch detection, periodic for full Opus retro
|
||||
'retro': 'periodic',
|
||||
'retro-base-branch': 'gate',
|
||||
|
||||
// Global discover
|
||||
'global-discover': 'gate',
|
||||
|
||||
// CSO — gate for security guardrails, periodic for quality
|
||||
'cso-full-audit': 'gate', // Hardcoded secrets detection
|
||||
'cso-diff-mode': 'gate',
|
||||
'cso-infra-scope': 'periodic',
|
||||
|
||||
// Learnings — gate (functional guardrail: seeded learnings must appear)
|
||||
'learnings-show': 'gate',
|
||||
|
||||
// Document-release — gate (CHANGELOG guardrail)
|
||||
'document-release': 'gate',
|
||||
|
||||
// Codex — periodic (Opus, requires codex CLI)
|
||||
'codex-review': 'periodic',
|
||||
|
||||
// Multi-AI — periodic (require external CLIs)
|
||||
'codex-discover-skill': 'periodic',
|
||||
'codex-review-findings': 'periodic',
|
||||
'gemini-smoke': 'periodic',
|
||||
|
||||
// Design — gate for cheap functional, periodic for Opus/quality
|
||||
'design-consultation-core': 'periodic',
|
||||
'design-consultation-existing': 'periodic',
|
||||
'design-consultation-research': 'gate',
|
||||
'design-consultation-preview': 'gate',
|
||||
'plan-design-review-no-ui-scope': 'gate',
|
||||
'design-review-fix': 'periodic',
|
||||
'design-shotgun-path': 'gate',
|
||||
'design-shotgun-session': 'gate',
|
||||
'design-shotgun-full': 'periodic',
|
||||
|
||||
// /diagram — triplet is deterministic functional, judge is a quality benchmark
|
||||
'diagram-triplet': 'gate',
|
||||
'diagram-authoring-quality': 'periodic',
|
||||
|
||||
// gstack-upgrade
|
||||
'gstack-upgrade-happy-path': 'gate',
|
||||
|
||||
// Deploy skills
|
||||
'land-and-deploy-workflow': 'gate',
|
||||
'land-and-deploy-first-run': 'gate',
|
||||
'land-and-deploy-review-gate': 'gate',
|
||||
'canary-workflow': 'gate',
|
||||
'benchmark-workflow': 'gate',
|
||||
'setup-deploy-workflow': 'gate',
|
||||
|
||||
// Autoplan — periodic (not yet implemented)
|
||||
'autoplan-core': 'periodic',
|
||||
'autoplan-dual-voice': 'periodic',
|
||||
|
||||
// Multi-provider benchmark — periodic (requires external CLIs + auth, paid)
|
||||
'benchmark-providers-live': 'periodic',
|
||||
|
||||
// Browser-skills Phase 2a — gate (D1/D3 contracts must not silently break)
|
||||
'scrape-match-path': 'gate',
|
||||
'scrape-prototype-path': 'gate',
|
||||
'skillify-happy-path': 'gate',
|
||||
'skillify-provenance-refusal': 'gate',
|
||||
'skillify-approval-reject': 'gate',
|
||||
|
||||
// Skill routing — periodic (LLM routing is non-deterministic)
|
||||
'journey-ideation': 'periodic',
|
||||
'journey-plan-eng': 'periodic',
|
||||
'journey-debug': 'periodic',
|
||||
'journey-qa': 'periodic',
|
||||
'journey-code-review': 'periodic',
|
||||
'journey-ship': 'periodic',
|
||||
'journey-docs': 'periodic',
|
||||
'journey-retro': 'periodic',
|
||||
'journey-design-system': 'periodic',
|
||||
'journey-visual-qa': 'periodic',
|
||||
|
||||
// Opus 4.7 overlay evals — periodic (non-deterministic LLM behavior + Opus cost)
|
||||
'fanout-arm-overlay-on': 'periodic',
|
||||
'fanout-arm-overlay-off': 'periodic',
|
||||
|
||||
// Overlay efficacy harness (SDK, paid) — periodic only
|
||||
'overlay-harness-opus-4-7-fanout-toy': 'periodic',
|
||||
'overlay-harness-opus-4-7-fanout-realistic': 'periodic',
|
||||
|
||||
// /ios-qa daemon + codegen — no-device path runs every PR (no hardware
|
||||
// dependency, deterministic). with-device path requires GSTACK_HAS_IOS_DEVICE.
|
||||
'ios-qa-e2e': 'gate',
|
||||
// Swift toolchain only, no device required, but heavier than TS unit tests.
|
||||
'ios-qa-swift-build': 'periodic',
|
||||
// Requires a real connected + paired iPhone. Manual-trigger only.
|
||||
'ios-qa-device': 'periodic',
|
||||
// /spec end-to-end PTY pipeline (paid, non-deterministic — periodic-tier).
|
||||
'spec-execute': 'periodic',
|
||||
};
|
||||
|
||||
/**
|
||||
* LLM-judge test touchfiles — keyed by test description string.
|
||||
*/
|
||||
export const LLM_JUDGE_TOUCHFILES: Record<string, string[]> = {
|
||||
'command reference table': ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts'],
|
||||
'snapshot flags reference': ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts'],
|
||||
'browse/SKILL.md reference': ['browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**'],
|
||||
'setup block': ['SKILL.md', 'SKILL.md.tmpl'],
|
||||
'regression vs baseline': ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json'],
|
||||
'qa/SKILL.md workflow': ['qa/SKILL.md', 'qa/SKILL.md.tmpl'],
|
||||
'qa/SKILL.md health rubric': ['qa/SKILL.md', 'qa/SKILL.md.tmpl'],
|
||||
'qa/SKILL.md anti-refusal': ['qa/SKILL.md', 'qa/SKILL.md.tmpl', 'qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'],
|
||||
'cross-skill greptile consistency': ['review/SKILL.md', 'review/SKILL.md.tmpl', 'ship/SKILL.md', 'ship/SKILL.md.tmpl', 'review/greptile-triage.md', 'retro/SKILL.md', 'retro/SKILL.md.tmpl'],
|
||||
'baseline score pinning': ['SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json'],
|
||||
|
||||
// Ship & Release
|
||||
'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl'],
|
||||
'document-release/SKILL.md workflow': ['document-release/SKILL.md', 'document-release/SKILL.md.tmpl'],
|
||||
|
||||
// Plan Reviews
|
||||
'plan-ceo-review/SKILL.md modes': ['plan-ceo-review/SKILL.md', 'plan-ceo-review/SKILL.md.tmpl'],
|
||||
'plan-eng-review/SKILL.md sections': ['plan-eng-review/SKILL.md', 'plan-eng-review/SKILL.md.tmpl'],
|
||||
|
||||
// /spec authored-spec quality (paid LLM-judge — periodic-tier).
|
||||
'plan-design-review/SKILL.md passes': ['plan-design-review/SKILL.md', 'plan-design-review/SKILL.md.tmpl'],
|
||||
|
||||
// Design skills
|
||||
'design-review/SKILL.md fix loop': ['design-review/SKILL.md', 'design-review/SKILL.md.tmpl'],
|
||||
'design-consultation/SKILL.md research': ['design-consultation/SKILL.md', 'design-consultation/SKILL.md.tmpl'],
|
||||
|
||||
// Office Hours
|
||||
'office-hours/SKILL.md spec review': ['office-hours/SKILL.md', 'office-hours/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'office-hours/SKILL.md design sketch': ['office-hours/SKILL.md', 'office-hours/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
// Deploy skills
|
||||
'land-and-deploy/SKILL.md workflow': ['land-and-deploy/SKILL.md', 'land-and-deploy/SKILL.md.tmpl'],
|
||||
'canary/SKILL.md monitoring loop': ['canary/SKILL.md', 'canary/SKILL.md.tmpl'],
|
||||
'benchmark/SKILL.md perf collection': ['benchmark/SKILL.md', 'benchmark/SKILL.md.tmpl'],
|
||||
'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl'],
|
||||
|
||||
// Other skills
|
||||
'retro/SKILL.md instructions': ['retro/SKILL.md', 'retro/SKILL.md.tmpl'],
|
||||
'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'],
|
||||
'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl'],
|
||||
|
||||
// Voice directive
|
||||
'voice directive tone': ['scripts/resolvers/preamble.ts', 'review/SKILL.md', 'review/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
};
|
||||
|
||||
/**
|
||||
* Changes to any of these files trigger ALL tests (both E2E and LLM-judge).
|
||||
* This module is split into three files:
|
||||
*
|
||||
* Keep this list minimal — only files that genuinely affect every test.
|
||||
* Scoped dependencies (gen-skill-docs, llm-judge, test-server, worktree,
|
||||
* codex/gemini session runners) belong in individual test entries instead.
|
||||
*/
|
||||
export const GLOBAL_TOUCHFILES = [
|
||||
'test/helpers/session-runner.ts', // All E2E tests use this runner
|
||||
'test/helpers/hermetic-env.ts', // Changes every E2E child's environment
|
||||
'test/helpers/eval-store.ts', // All E2E tests store results here
|
||||
'test/helpers/touchfiles.ts', // Self-referential — reclassifying wrong is dangerous
|
||||
];
|
||||
|
||||
// --- Base branch detection ---
|
||||
|
||||
/**
|
||||
* Detect the base branch by trying refs in order.
|
||||
* Returns the first valid ref, or null if none found.
|
||||
*/
|
||||
export function detectBaseBranch(cwd: string): string | null {
|
||||
for (const ref of ['origin/main', 'origin/master', 'main', 'master']) {
|
||||
const result = spawnSync('git', ['rev-parse', '--verify', ref], {
|
||||
cwd, stdio: 'pipe', timeout: 3000,
|
||||
});
|
||||
if (result.status === 0) return ref;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get list of files changed between base branch and HEAD.
|
||||
*/
|
||||
export function getChangedFiles(baseBranch: string, cwd: string): string[] {
|
||||
const result = spawnSync('git', ['diff', '--name-only', `${baseBranch}...HEAD`], {
|
||||
cwd, stdio: 'pipe', timeout: 5000,
|
||||
});
|
||||
if (result.status !== 0) return [];
|
||||
return result.stdout.toString().trim().split('\n').filter(Boolean);
|
||||
}
|
||||
|
||||
// --- Test selection ---
|
||||
|
||||
/**
|
||||
* Select tests to run based on changed files.
|
||||
* - ./touchfiles-data.ts — the four touchfile/tier maps (E2E_TOUCHFILES,
|
||||
* E2E_TIERS, LLM_JUDGE_TOUCHFILES, GLOBAL_TOUCHFILES), LITERALS ONLY.
|
||||
* Map-diff selection evaluates OLD git versions of that file standalone
|
||||
* to diff the maps across commits, so it must stay importable pure data
|
||||
* (zero imports, zero executable logic).
|
||||
* - ./test-selection.ts — the logic: matchGlob, detectBaseBranch,
|
||||
* getChangedFiles, selectTests.
|
||||
* - this facade — re-exports everything so the ~dozen existing import
|
||||
* sites (e2e-helpers, eval-select, e2e-tier-alignment, paid-test-set,
|
||||
* the *-e2e test files, …) keep importing from './touchfiles' unchanged.
|
||||
*
|
||||
* Algorithm:
|
||||
* 1. If any changed file matches a global touchfile → run ALL tests
|
||||
* 2. Otherwise, for each test, check if any changed file matches its patterns
|
||||
* 3. Return selected + skipped lists with reason
|
||||
* test/touchfiles-facade.test.ts pins the shape: the literal-only tripwire
|
||||
* on the data file, and export parity (every export of both halves is
|
||||
* re-exported here by identity).
|
||||
*/
|
||||
export function selectTests(
|
||||
changedFiles: string[],
|
||||
touchfiles: Record<string, string[]>,
|
||||
globalTouchfiles: string[] = GLOBAL_TOUCHFILES,
|
||||
): { selected: string[]; skipped: string[]; reason: string } {
|
||||
const allTestNames = Object.keys(touchfiles);
|
||||
|
||||
// Global touchfile hit → run all
|
||||
for (const file of changedFiles) {
|
||||
if (globalTouchfiles.some(g => matchGlob(file, g))) {
|
||||
return { selected: allTestNames, skipped: [], reason: `global: ${file}` };
|
||||
}
|
||||
}
|
||||
export {
|
||||
E2E_TOUCHFILES,
|
||||
E2E_TIERS,
|
||||
LLM_JUDGE_TOUCHFILES,
|
||||
GLOBAL_TOUCHFILES,
|
||||
} from './touchfiles-data';
|
||||
|
||||
// Per-test matching
|
||||
const selected: string[] = [];
|
||||
const skipped: string[] = [];
|
||||
for (const [testName, patterns] of Object.entries(touchfiles)) {
|
||||
const hit = changedFiles.some(f => patterns.some(p => matchGlob(f, p)));
|
||||
(hit ? selected : skipped).push(testName);
|
||||
}
|
||||
export {
|
||||
matchGlob,
|
||||
detectBaseBranch,
|
||||
getChangedFiles,
|
||||
selectTests,
|
||||
diffTouchfileMaps,
|
||||
diffTouchfileMapsCore,
|
||||
TOUCHFILES_DATA_PATH,
|
||||
} from './test-selection';
|
||||
|
||||
return { selected, skipped, reason: 'diff' };
|
||||
}
|
||||
export type {
|
||||
TouchfileMaps,
|
||||
MapDiffCause,
|
||||
MapDiffOutcome,
|
||||
} from './test-selection';
|
||||
|
||||
@@ -401,7 +401,9 @@ describe('host-config-export.ts CLI', () => {
|
||||
expect(exitCode).toBe(1);
|
||||
});
|
||||
|
||||
test('detect finds claude (since we are running in claude)', () => {
|
||||
// Gated: the secretless free-tests CI lane deliberately installs no claude
|
||||
// CLI, so "we are running in claude" is false there by design.
|
||||
test.skipIf(!Bun.which('claude'))('detect finds claude (since we are running in claude)', () => {
|
||||
const { stdout, exitCode } = run('detect');
|
||||
expect(exitCode).toBe(0);
|
||||
// claude binary should be on PATH in this environment
|
||||
|
||||
@@ -14,10 +14,16 @@ import {
|
||||
PAID_TEST_GLOBS,
|
||||
classifyPaidTestFile,
|
||||
collectPaidTestFiles,
|
||||
computePaidDiffSelection,
|
||||
diffSkipDecisionForFile,
|
||||
formatSummary,
|
||||
isPaidTestFile,
|
||||
knownTestNamesInSource,
|
||||
partitionShardsByDiffSelection,
|
||||
planPaidShards,
|
||||
runPaidShards,
|
||||
summarize,
|
||||
summaryExitCode,
|
||||
type ShardOutcome,
|
||||
} from '../scripts/test-paid-shards';
|
||||
|
||||
@@ -28,6 +34,9 @@ describe('paid test enumeration', () => {
|
||||
expect(isPaidTestFile('test/codex-e2e.test.ts')).toBe(true);
|
||||
expect(isPaidTestFile('test/skill-e2e-triage-audit.test.ts')).toBe(true);
|
||||
// Outside the globs: no dash, extra suffix, or a free test.
|
||||
// 'test/skill-e2e.test.ts' is the DELETED pre-split monolith's name,
|
||||
// kept here as a regression pin: its glob-invisibility is exactly how
|
||||
// two gate tests went unexecuted for ~8 releases before the rehoming.
|
||||
expect(isPaidTestFile('test/skill-e2e.test.ts')).toBe(false);
|
||||
expect(isPaidTestFile('test/codex-e2e-recommendation-substance.test.ts')).toBe(false);
|
||||
expect(isPaidTestFile('test/paid-shards.test.ts')).toBe(false);
|
||||
@@ -129,3 +138,140 @@ describe('shard execution', () => {
|
||||
expect(summary).toMatchObject({ total: 2, executed: 1, passed: 1, neverStarted: 1 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('parent-side diff shard skipping', () => {
|
||||
const ALL_NAMES = ['alpha-test', 'beta-test', 'gamma-registered'];
|
||||
const TOUCHFILES: Record<string, string[]> = {
|
||||
'alpha-test': ['a/**'],
|
||||
'beta-test': ['b/**'],
|
||||
'gamma-registered': ['g/**', 'test/skill-e2e-gamma.test.ts'],
|
||||
};
|
||||
const SOURCES: Record<string, string> = {
|
||||
'test/skill-e2e-alpha.test.ts': "runSkillTest('alpha-test', async () => {});",
|
||||
'test/skill-e2e-beta.test.ts': 'describeIfSelected("beta", ["beta-test"], () => {});',
|
||||
// Constructed testName — invisible by quotes, mapped only via registration.
|
||||
'test/skill-e2e-gamma.test.ts': 'const name = buildName(); test(name, async () => {});',
|
||||
// No recognizable names, no registration — the fail-open class.
|
||||
'test/skill-e2e-opaque.test.ts': "const shouldRun = process.env.EVALS_TIER === 'periodic';",
|
||||
'test/codex-e2e.test.ts': 'codex tests keyed off CODEX_E2E_TOUCHFILES',
|
||||
};
|
||||
const opts = {
|
||||
readSource: (file: string) => {
|
||||
if (!(file in SOURCES)) throw new Error(`unreadable: ${file}`);
|
||||
return SOURCES[file];
|
||||
},
|
||||
allNames: ALL_NAMES,
|
||||
e2eTouchfiles: TOUCHFILES,
|
||||
};
|
||||
|
||||
test('knownTestNamesInSource matches only exact quoted strings', () => {
|
||||
expect(knownTestNamesInSource("x 'alpha-test' y", ['alpha-test', 'beta-test'])).toEqual(['alpha-test']);
|
||||
expect(knownTestNamesInSource('x "beta-test" y', ['alpha-test', 'beta-test'])).toEqual(['beta-test']);
|
||||
expect(knownTestNamesInSource('`alpha-test`', ['alpha-test'])).toEqual(['alpha-test']);
|
||||
// Substring inside a longer quoted string is not a hit.
|
||||
expect(knownTestNamesInSource("'alpha-test-extended'", ['alpha-test'])).toEqual([]);
|
||||
});
|
||||
|
||||
test('selected name in file → shard kept', () => {
|
||||
const d = diffSkipDecisionForFile('test/skill-e2e-alpha.test.ts', new Set(['alpha-test']), opts);
|
||||
expect(d.kept).toBe(true);
|
||||
expect(d.reason).toContain('alpha-test');
|
||||
});
|
||||
|
||||
test('no selected names in file → skipped-by-diff', () => {
|
||||
const d = diffSkipDecisionForFile('test/skill-e2e-beta.test.ts', new Set(['alpha-test']), opts);
|
||||
expect(d.kept).toBe(false);
|
||||
expect(d.reason).toContain('mapped test(s)');
|
||||
});
|
||||
|
||||
test('dep-list registration maps files with constructed test names', () => {
|
||||
const selected = diffSkipDecisionForFile('test/skill-e2e-gamma.test.ts', new Set(['gamma-registered']), opts);
|
||||
expect(selected.kept).toBe(true);
|
||||
const unselected = diffSkipDecisionForFile('test/skill-e2e-gamma.test.ts', new Set(['alpha-test']), opts);
|
||||
expect(unselected.kept).toBe(false);
|
||||
});
|
||||
|
||||
test('FAIL-OPEN: unmapped file kept, child self-skip authoritative', () => {
|
||||
const d = diffSkipDecisionForFile('test/skill-e2e-opaque.test.ts', new Set(['alpha-test']), opts);
|
||||
expect(d.kept).toBe(true);
|
||||
expect(d.reason).toContain('fail-open');
|
||||
});
|
||||
|
||||
test('FAIL-OPEN: unreadable source kept', () => {
|
||||
const d = diffSkipDecisionForFile('test/skill-e2e-missing.test.ts', new Set(['alpha-test']), opts);
|
||||
expect(d.kept).toBe(true);
|
||||
expect(d.reason).toContain('fail-open');
|
||||
});
|
||||
|
||||
test('FAIL-OPEN: non-skill-e2e paid files always kept', () => {
|
||||
const d = diffSkipDecisionForFile('test/codex-e2e.test.ts', new Set(['alpha-test']), opts);
|
||||
expect(d.kept).toBe(true);
|
||||
expect(d.reason).toContain('non-skill-e2e');
|
||||
});
|
||||
|
||||
test('run-all selection (null) bypasses skipping entirely', () => {
|
||||
const shards = [['test/skill-e2e-alpha.test.ts'], ['test/skill-e2e-beta.test.ts']];
|
||||
const { runnable, skipped } = partitionShardsByDiffSelection(shards, null, opts);
|
||||
expect(runnable).toEqual(shards);
|
||||
expect(skipped).toEqual([]);
|
||||
});
|
||||
|
||||
test('EVALS_ALL=1 yields run-all selection (no git consulted)', () => {
|
||||
const selection = computePaidDiffSelection({ EVALS_ALL: '1' } as NodeJS.ProcessEnv);
|
||||
expect(selection.selectedNames).toBeNull();
|
||||
expect(selection.reason).toContain('EVALS_ALL=1');
|
||||
expect(selection.totalTests).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('partition drops only all-skippable shards', () => {
|
||||
const shards = [
|
||||
['test/skill-e2e-alpha.test.ts'],
|
||||
['test/skill-e2e-beta.test.ts'],
|
||||
['test/skill-e2e-opaque.test.ts'],
|
||||
['test/codex-e2e.test.ts'],
|
||||
];
|
||||
const { runnable, skipped } = partitionShardsByDiffSelection(shards, new Set(['alpha-test']), opts);
|
||||
expect(runnable).toEqual([
|
||||
['test/skill-e2e-alpha.test.ts'],
|
||||
['test/skill-e2e-opaque.test.ts'],
|
||||
['test/codex-e2e.test.ts'],
|
||||
]);
|
||||
expect(skipped.length).toBe(1);
|
||||
expect(skipped[0].files).toEqual(['test/skill-e2e-beta.test.ts']);
|
||||
});
|
||||
|
||||
test('taxonomy: skipped-by-diff counted separately, never conflated with never-started', () => {
|
||||
const summary = summarize([
|
||||
{ shard: 1, files: ['a'], status: 'passed', exitCode: 0, elapsedMs: 1, groupPid: 1 },
|
||||
{ shard: 2, files: ['b'], status: 'skipped-by-diff', exitCode: null, elapsedMs: 0, groupPid: null },
|
||||
{ shard: 3, files: ['c'], status: 'never-started', exitCode: null, elapsedMs: 0, groupPid: null },
|
||||
]);
|
||||
expect(summary).toMatchObject({
|
||||
total: 3, executed: 1, passed: 1, skippedByDiff: 1, neverStarted: 1,
|
||||
});
|
||||
const lines = formatSummary(summary);
|
||||
expect(lines[1]).toContain('1 skipped by diff');
|
||||
expect(lines[1]).toContain('1 never started');
|
||||
expect(lines.some((l) => l.includes('skipped-by-diff') && l.includes('b'))).toBe(true);
|
||||
});
|
||||
|
||||
test('exit code ignores skipped-by-diff shards (they are successes)', () => {
|
||||
const allGood = summarize([
|
||||
{ shard: 1, files: ['a'], status: 'passed', exitCode: 0, elapsedMs: 1, groupPid: 1 },
|
||||
{ shard: 2, files: ['b'], status: 'skipped-by-diff', exitCode: null, elapsedMs: 0, groupPid: null },
|
||||
]);
|
||||
expect(summaryExitCode(allGood)).toBe(0);
|
||||
|
||||
const withFailure = summarize([
|
||||
{ shard: 1, files: ['a'], status: 'failed', exitCode: 1, elapsedMs: 1, groupPid: 1 },
|
||||
{ shard: 2, files: ['b'], status: 'skipped-by-diff', exitCode: null, elapsedMs: 0, groupPid: null },
|
||||
]);
|
||||
expect(summaryExitCode(withFailure)).toBe(1);
|
||||
|
||||
const withNeverStarted = summarize([
|
||||
{ shard: 1, files: ['a'], status: 'never-started', exitCode: null, elapsedMs: 0, groupPid: null },
|
||||
{ shard: 2, files: ['b'], status: 'skipped-by-diff', exitCode: null, elapsedMs: 0, groupPid: null },
|
||||
]);
|
||||
expect(summaryExitCode(withNeverStarted)).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -104,6 +104,36 @@ describe("HIGH credential patterns", () => {
|
||||
test("db.url_with_password flags real password, skips placeholder/env-var", () => {
|
||||
expect(ids("postgres://user:s3cretP@ss@db.example.com/app")).toContain("db.url_with_password");
|
||||
expect(ids("postgres://user:${DB_PASSWORD}@host/app")).not.toContain("db.url_with_password");
|
||||
// Literal PASSWORD placeholder (URL-format doc comments).
|
||||
expect(ids("postgresql://USER:PASSWORD@host/db")).not.toContain("db.url_with_password");
|
||||
// JS template interpolations are code, not credentials — the
|
||||
// uppercase-only placeholder form blocked a push over
|
||||
// `postgresql://${dbUser}:${dbPass}@...` in a bash->TS port.
|
||||
// eslint-disable-next-line no-template-curly-in-string
|
||||
expect(ids("postgresql://${dbUser}:${dbPass}@${dbHost}:5432/db")).not.toContain("db.url_with_password");
|
||||
// Assembled at runtime so this file's own diff never contains a
|
||||
// credential-shaped literal (the prepush guard scans exact pushed bytes).
|
||||
expect(ids("postgres://admin:" + "hun" + "ter2@db.internal/app")).toContain("db.url_with_password");
|
||||
// Bare $UPPER_SNAKE is shell convention → suppressed; bare $lowercase is
|
||||
// NOT an interpolation form — a real password starting with `$` must
|
||||
// still block (both-braces-optional would have let it through).
|
||||
expect(ids("postgres://user:$DB_PASSWORD@host/app")).not.toContain("db.url_with_password");
|
||||
expect(ids("postgres://admin:$" + "hun" + "ter2@db.internal/app")).toContain("db.url_with_password");
|
||||
// Mismatched brace is not an interpolation either (assembled at runtime
|
||||
// so this file's own pushed bytes carry no blockable URL shape).
|
||||
expect(ids("postgres://admin:${" + "dbPass@db.internal/app")).toContain("db.url_with_password");
|
||||
// A fully-braced interpolation is code whatever it contains — the DSN
|
||||
// builder's `${encodeURIComponent(dbPass)}` call site must not scan as a
|
||||
// pushed secret.
|
||||
expect(ids("postgresql://user:${encodeURIComponent(dbPass)}@host:5432/db")).not.toContain("db.url_with_password");
|
||||
// A LOWERCASE literal 'password'/'pass' at the URL-password position is a
|
||||
// real (terrible) credential, not a doc placeholder — only the ALL-CAPS
|
||||
// doc convention (USER:PASSWORD) is suppressed. Assembled at runtime so
|
||||
// this file's own bytes never carry a live credential shape.
|
||||
expect(ids("postgres://admin:" + "pass" + "word@10.0.0.5/app")).toContain("db.url_with_password");
|
||||
expect(ids("https://root:" + "pa" + "ss@127.0.0.1/")).toContain("creds.basic_auth_url");
|
||||
// Structural placeholders still suppress at the URL position.
|
||||
expect(ids("postgres://user:<your-password>@host/db")).not.toContain("db.url_with_password");
|
||||
});
|
||||
|
||||
test("all HIGH patterns block (exit 3)", () => {
|
||||
|
||||
+9
-1
@@ -25,7 +25,15 @@ function run(cmd: string, env: Record<string, string> = {}, expectFail = false):
|
||||
try {
|
||||
return execSync(cmd, {
|
||||
cwd: ROOT,
|
||||
env: { ...process.env, GSTACK_STATE_DIR: tmpDir, ...env },
|
||||
// A sibling test file in the same shard PROCESS can leave GSTACK_HOME
|
||||
// set on process.env; relink/config children must resolve state ONLY
|
||||
// via the dirs this test passes (observed: 'fresh install' test saw a
|
||||
// neighbor's skill_prefix and produced prefixed names).
|
||||
env: (() => {
|
||||
const child: Record<string, string | undefined> = { ...process.env, GSTACK_STATE_DIR: tmpDir, ...env };
|
||||
if (!('GSTACK_HOME' in env)) delete child.GSTACK_HOME;
|
||||
return child;
|
||||
})(),
|
||||
encoding: 'utf-8',
|
||||
timeout: 10000,
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
|
||||
@@ -19,6 +19,7 @@ import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { SKILL_COVERAGE } from './skill-coverage-matrix';
|
||||
import { skillCensus } from './helpers/skill-census';
|
||||
|
||||
const REPO_ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
@@ -31,46 +32,14 @@ function readSkillMd(skill: string): string | null {
|
||||
}
|
||||
}
|
||||
|
||||
function listSkillDirs(): string[] {
|
||||
const entries = fs.readdirSync(REPO_ROOT, { withFileTypes: true });
|
||||
return entries
|
||||
.filter(e => e.isDirectory() && !e.name.startsWith('.'))
|
||||
.filter(e => e.name !== 'node_modules' && e.name !== 'docs' && e.name !== 'test')
|
||||
.filter(e => fs.existsSync(path.join(REPO_ROOT, e.name, 'SKILL.md')))
|
||||
.map(e => e.name)
|
||||
.sort();
|
||||
}
|
||||
// Registry-completeness assertions ("every skill on disk is registered",
|
||||
// "every entry has a gate test") live in test/skill-coverage-matrix.test.ts —
|
||||
// they were duplicated here with a DIFFERENT hand-rolled directory walk, which
|
||||
// is the divergence class test/helpers/skill-census.ts exists to kill. This
|
||||
// file owns the per-skill structural compliance checks only.
|
||||
|
||||
describe('skill-coverage-floor: every skill passes structural compliance', () => {
|
||||
const skills = listSkillDirs();
|
||||
|
||||
test('skill registry mentions every skill on disk', () => {
|
||||
const onDisk = new Set(skills);
|
||||
const inRegistry = new Set(Object.keys(SKILL_COVERAGE));
|
||||
const missingFromRegistry: string[] = [];
|
||||
for (const s of onDisk) {
|
||||
if (!inRegistry.has(s)) missingFromRegistry.push(s);
|
||||
}
|
||||
if (missingFromRegistry.length > 0) {
|
||||
throw new Error(
|
||||
`Skills on disk missing from test/skill-coverage-matrix.ts: ${missingFromRegistry.join(', ')}. ` +
|
||||
`Add an entry to SKILL_COVERAGE with at least 'test/skill-coverage-floor.test.ts' in gate[].`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('every registry entry has at least one gate-tier test', () => {
|
||||
const missingGate: string[] = [];
|
||||
for (const [skill, coverage] of Object.entries(SKILL_COVERAGE)) {
|
||||
if (!coverage.gate || coverage.gate.length === 0) missingGate.push(skill);
|
||||
}
|
||||
if (missingGate.length > 0) {
|
||||
throw new Error(
|
||||
`Skills with no gate-tier eval: ${missingGate.join(', ')}. ` +
|
||||
`Eval-first foundation requires at least one CI-blocking check per skill.`,
|
||||
);
|
||||
}
|
||||
});
|
||||
const skills = skillCensus(REPO_ROOT).authoredSkills;
|
||||
|
||||
test('every gate-tier test path referenced in registry exists on disk', () => {
|
||||
const missing: string[] = [];
|
||||
|
||||
@@ -8,18 +8,17 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { SKILL_COVERAGE, type SkillCoverage } from './skill-coverage-matrix';
|
||||
import { skillCensus } from './helpers/skill-census';
|
||||
|
||||
const REPO_ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
// Canonical walk (skill-census.ts). This file and skill-coverage-floor
|
||||
// previously hand-rolled two DIFFERENT walks (one skipped node_modules/docs/
|
||||
// test, one didn't) — exactly the divergence class the census exists to kill.
|
||||
function discoverSkills(): string[] {
|
||||
return fs.readdirSync(REPO_ROOT, { withFileTypes: true })
|
||||
.filter(e => e.isDirectory() && !e.name.startsWith('.'))
|
||||
.filter(e => fs.existsSync(path.join(REPO_ROOT, e.name, 'SKILL.md')))
|
||||
.map(e => e.name)
|
||||
.sort();
|
||||
return skillCensus(REPO_ROOT).authoredSkills;
|
||||
}
|
||||
|
||||
describe('skill coverage matrix', () => {
|
||||
@@ -29,16 +28,23 @@ describe('skill coverage matrix', () => {
|
||||
});
|
||||
|
||||
test('every entry has the right shape', () => {
|
||||
const missingGate: string[] = [];
|
||||
for (const [skill, coverage] of Object.entries(SKILL_COVERAGE)) {
|
||||
expect(Array.isArray(coverage.gate)).toBe(true);
|
||||
expect(Array.isArray(coverage.periodic)).toBe(true);
|
||||
expect(coverage.gate.length).toBeGreaterThan(0);
|
||||
if (!coverage.gate || coverage.gate.length === 0) missingGate.push(skill);
|
||||
for (const p of [...coverage.gate, ...coverage.periodic]) {
|
||||
expect(typeof p).toBe('string');
|
||||
expect(p.startsWith('test/')).toBe(true);
|
||||
expect(p.endsWith('.test.ts')).toBe(true);
|
||||
}
|
||||
}
|
||||
if (missingGate.length > 0) {
|
||||
throw new Error(
|
||||
`Skills with no gate-tier eval: ${missingGate.join(', ')}. ` +
|
||||
`Eval-first foundation requires at least one CI-blocking check per skill.`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('every skill on disk has a registry entry', () => {
|
||||
|
||||
@@ -67,6 +67,8 @@ interface MatrixSkill {
|
||||
skill: string;
|
||||
fixtures: Record<string, string>;
|
||||
scenario: string;
|
||||
/** D1a regressor pin: explicit capture model when the Sonnet default measurably fails this entry. */
|
||||
model?: string;
|
||||
}
|
||||
|
||||
const MATRIX: MatrixSkill[] = [
|
||||
@@ -99,6 +101,13 @@ const MATRIX: MatrixSkill[] = [
|
||||
skill: 'spec',
|
||||
fixtures: {},
|
||||
scenario: 'Turn this vague intent into a precise spec: "add email notifications when a task is assigned to someone." Walk the spec workflow until the first AskUserQuestion.',
|
||||
// D1a pin-on-regressors, with receipts (2026-08-16 re-baseline): under
|
||||
// the Sonnet capture default this entry failed twice ("never reached a
|
||||
// question in budget", 242s) while the six sibling entries passed; the
|
||||
// controlled Opus re-run passed cleanly (7/7 format, substance 5, 160s).
|
||||
// The spec workflow's long pre-question phase needs the stronger model
|
||||
// to reach its first AskUserQuestion inside the turn budget.
|
||||
model: 'claude-opus-4-7',
|
||||
},
|
||||
{
|
||||
skill: 'design-consultation',
|
||||
@@ -130,6 +139,7 @@ describeE2E('AUQ behavioral matrix (periodic)', () => {
|
||||
scenario: m.scenario,
|
||||
testName: `auq-matrix-${m.skill}`,
|
||||
runId,
|
||||
model: m.model,
|
||||
});
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
logCost, recordE2E,
|
||||
createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { extractSkillBody } from './helpers/skill-fixture';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
@@ -43,11 +44,14 @@ function setupWorkdir(suffix: string): { workDir: string; gstackHome: string; sl
|
||||
run('git', ['commit', '-m', 'initial']);
|
||||
|
||||
// Install skills into .claude/skills/ for claude -p auto-discovery.
|
||||
// The tests exercise the full save/restore/list flows, so keep the whole
|
||||
// skill-specific body but drop the ~780-line shared preamble the tests
|
||||
// never touch (CLAUDE.md: "E2E test fixtures: extract, don't copy").
|
||||
const skillsDir = path.join(workDir, '.claude', 'skills');
|
||||
for (const skill of ['context-save', 'context-restore']) {
|
||||
const destDir = path.join(skillsDir, skill);
|
||||
fs.mkdirSync(destDir, { recursive: true });
|
||||
fs.copyFileSync(path.join(ROOT, skill, 'SKILL.md'), path.join(destDir, 'SKILL.md'));
|
||||
fs.writeFileSync(path.join(destDir, 'SKILL.md'), extractSkillBody(path.join(ROOT, skill)));
|
||||
}
|
||||
|
||||
// Install the bin scripts referenced by the preamble.
|
||||
@@ -443,14 +447,15 @@ Do NOT use AskUserQuestion.`,
|
||||
// Broad surface: the list output may only appear in bash tool_result
|
||||
// entries (find output, file reads) rather than the agent's final text.
|
||||
const out = fullOutputSurface(result);
|
||||
// Must show the main-branch save. Hide the other branches' saves.
|
||||
// Match by filename timestamp (stable, unambiguous) plus a looser
|
||||
// prose check.
|
||||
// Must show the main-branch save. Match by filename timestamp (stable,
|
||||
// unambiguous) plus a looser prose check.
|
||||
const showsMain = /20260101-120000|main-work/.test(out);
|
||||
// Hide checks scope to the FINAL text output only: fullOutputSurface
|
||||
// includes bash tool_results, and a legitimate `ls` of the checkpoint
|
||||
// dir lists every branch's filename. The filtering under test happens
|
||||
// in the user-facing list, not in the agent's intermediate reads.
|
||||
// The hide-assertions scan the FINAL TEXT only. This test went 0-for-26
|
||||
// ($5.28 burned, zero passes) because they used the broad surface: any
|
||||
// agent that ran `ls` on the checkpoints dir — the natural first step of
|
||||
// a list flow — surfaced all three filenames in a tool_result and failed,
|
||||
// even when its user-facing listing filtered correctly. What must hide
|
||||
// the other branches is the LISTING the user sees, not the agent's eyes.
|
||||
const finalText = result.output ?? '';
|
||||
const hidesAlpha = !/20260202-120000|LISTCURR_ALPHA_TOKEN/.test(finalText);
|
||||
const hidesBeta = !/20260303-120000|LISTCURR_BETA_TOKEN/.test(finalText);
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
/**
|
||||
* Coverage-audit E2E — /review and /plan-eng-review coverage-diagram flows.
|
||||
*
|
||||
* Rehomed VERBATIM from the pre-split monolith (test/skill-e2e.test.ts,
|
||||
* deleted on this branch): the monolith's filename never matched the paid
|
||||
* glob (`test/skill-e2e-*.test.ts` — note the hyphen), so these two GATE-tier
|
||||
* tests (`review-coverage-audit`, `plan-eng-coverage-audit` in E2E_TIERS)
|
||||
* silently never executed after the v1.56 split.
|
||||
*
|
||||
* DRIFT WARNING (attribution for the first paid run after rehoming): the
|
||||
* prompts reference "Step 4.75 (Test Coverage Diagram)" in review/SKILL.md
|
||||
* and a "Test Coverage Audit" section in plan-eng-review/SKILL.md. NEITHER
|
||||
* section exists in the current generated skills — the skills drifted while
|
||||
* these tests were zombies. Test bodies are copied faithfully (no behavioral
|
||||
* edits), so a failure here indicts the ~8 releases of drift, not the move.
|
||||
* The only change vs the monolith bodies: the staged SKILL.md fixtures are
|
||||
* extracted via test/helpers/skill-fixture.ts (extractSkillBody — full
|
||||
* skill-specific body, shared preamble dropped) per CLAUDE.md
|
||||
* "E2E test fixtures: extract, don't copy".
|
||||
*/
|
||||
|
||||
import { test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
describeIfSelected,
|
||||
copyDirSync, logCost, recordE2E,
|
||||
createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { extractSkillBody } from './helpers/skill-fixture';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
const evalCollector = createEvalCollector('e2e-coverage-audit');
|
||||
|
||||
// --- Review Coverage Audit E2E ---
|
||||
|
||||
describeIfSelected('Review Coverage Audit E2E', ['review-coverage-audit'], () => {
|
||||
let reviewCoverageDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
reviewCoverageDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-review-coverage-'));
|
||||
|
||||
// Copy review skill files, then replace the SKILL.md with the extracted
|
||||
// skill body (extract, don't copy — the checklists/specialists in the
|
||||
// dir are small hand-written files and stay whole).
|
||||
copyDirSync(path.join(ROOT, 'review'), path.join(reviewCoverageDir, 'review'));
|
||||
fs.writeFileSync(
|
||||
path.join(reviewCoverageDir, 'review', 'SKILL.md'),
|
||||
extractSkillBody(path.join(ROOT, 'review')),
|
||||
);
|
||||
|
||||
// Use shared fixture for billing project with coverage gaps
|
||||
const { createCoverageAuditFixture } = require('./fixtures/coverage-audit-fixture');
|
||||
createCoverageAuditFixture(reviewCoverageDir);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(reviewCoverageDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
test('/review Step 4.75 produces coverage diagram', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read the file review/SKILL.md for the review workflow instructions.
|
||||
|
||||
You are on the feature/billing branch. The base branch is main.
|
||||
This is a test project — there is no remote, no PR to create.
|
||||
|
||||
ONLY run Step 4.75 (Test Coverage Diagram) from the review workflow.
|
||||
Skip all other steps (scope drift, checklist, design review, fix-first, etc.).
|
||||
|
||||
The source code is in ${reviewCoverageDir}/src/billing.ts.
|
||||
Existing tests are in ${reviewCoverageDir}/test/billing.test.ts.
|
||||
|
||||
Produce the ASCII coverage diagram showing which code paths are tested and which have gaps.
|
||||
Output the diagram directly.`,
|
||||
workingDirectory: reviewCoverageDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
|
||||
timeout: 120_000,
|
||||
testName: 'review-coverage-audit',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/review coverage audit', result);
|
||||
recordE2E(evalCollector, '/review Step 4.75 coverage audit', 'Review Coverage Audit E2E', result, {
|
||||
passed: result.exitReason === 'success',
|
||||
});
|
||||
|
||||
expect(result.exitReason).toBe('success');
|
||||
|
||||
// Check output contains coverage diagram elements
|
||||
const output = result.output || '';
|
||||
const outputLower = output.toLowerCase();
|
||||
const hasGap = outputLower.includes('gap') || outputLower.includes('no test');
|
||||
const hasTested = outputLower.includes('tested') || output.includes('✓') || output.includes('★');
|
||||
const hasCoverage = outputLower.includes('coverage') || outputLower.includes('paths tested');
|
||||
|
||||
console.log(`Output has GAP markers: ${hasGap}`);
|
||||
console.log(`Output has TESTED markers: ${hasTested}`);
|
||||
console.log(`Output has coverage summary: ${hasCoverage}`);
|
||||
|
||||
// The agent MUST produce a coverage diagram with gap and tested markers
|
||||
expect(hasGap || hasTested).toBe(true);
|
||||
|
||||
// At minimum, the agent should have read the source and test files
|
||||
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
|
||||
expect(readCalls.length).toBeGreaterThan(0);
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
// --- Plan Eng Review Coverage Audit E2E ---
|
||||
|
||||
describeIfSelected('Plan Eng Review Coverage Audit E2E', ['plan-eng-coverage-audit'], () => {
|
||||
let planCoverageDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
planCoverageDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-coverage-'));
|
||||
|
||||
// Copy plan-eng-review skill files, then replace the SKILL.md with the
|
||||
// extracted skill body (extract, don't copy).
|
||||
copyDirSync(path.join(ROOT, 'plan-eng-review'), path.join(planCoverageDir, 'plan-eng-review'));
|
||||
fs.writeFileSync(
|
||||
path.join(planCoverageDir, 'plan-eng-review', 'SKILL.md'),
|
||||
extractSkillBody(path.join(ROOT, 'plan-eng-review')),
|
||||
);
|
||||
|
||||
// Use shared fixture for billing project with coverage gaps
|
||||
const { createCoverageAuditFixture } = require('./fixtures/coverage-audit-fixture');
|
||||
createCoverageAuditFixture(planCoverageDir);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(planCoverageDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
test('/plan-eng-review coverage audit traces plan codepaths', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read the file plan-eng-review/SKILL.md for the plan review workflow instructions.
|
||||
|
||||
You are on the feature/billing branch. The base branch is main.
|
||||
This is a test project — there is no remote, no PR to create.
|
||||
|
||||
ONLY run the Test Coverage Audit section from the plan review workflow.
|
||||
Skip all other steps (architecture, code quality, performance, etc.).
|
||||
|
||||
The source code is in ${planCoverageDir}/src/billing.ts.
|
||||
Existing tests are in ${planCoverageDir}/test/billing.test.ts.
|
||||
|
||||
Produce the ASCII coverage diagram showing which code paths are tested and which have gaps.
|
||||
Output the diagram directly.`,
|
||||
workingDirectory: planCoverageDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
|
||||
timeout: 120_000,
|
||||
testName: 'plan-eng-coverage-audit',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/plan-eng-review coverage audit', result);
|
||||
recordE2E(evalCollector, '/plan-eng-review coverage audit', 'Plan Eng Review Coverage Audit E2E', result, {
|
||||
passed: result.exitReason === 'success',
|
||||
});
|
||||
|
||||
expect(result.exitReason).toBe('success');
|
||||
|
||||
// Check output contains coverage diagram elements
|
||||
const output = result.output || '';
|
||||
const outputLower = output.toLowerCase();
|
||||
const hasGap = outputLower.includes('gap') || outputLower.includes('no test');
|
||||
const hasTested = outputLower.includes('tested') || output.includes('✓') || output.includes('★');
|
||||
const hasCoverage = outputLower.includes('coverage') || outputLower.includes('paths tested');
|
||||
|
||||
console.log(`Output has GAP markers: ${hasGap}`);
|
||||
console.log(`Output has TESTED markers: ${hasTested}`);
|
||||
console.log(`Output has coverage summary: ${hasCoverage}`);
|
||||
|
||||
// The agent MUST produce a coverage diagram with gap and tested markers
|
||||
expect(hasGap || hasTested).toBe(true);
|
||||
|
||||
// At minimum, the agent should have read the source and test files
|
||||
const readCalls = result.toolCalls.filter(tc => tc.tool === 'Read');
|
||||
expect(readCalls.length).toBeGreaterThan(0);
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
afterAll(async () => {
|
||||
await finalizeEvalCollector(evalCollector);
|
||||
});
|
||||
@@ -20,6 +20,7 @@
|
||||
import { describe, test, expect, afterAll } from 'bun:test';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import { EvalCollector } from './helpers/eval-store';
|
||||
import { extractSkillHead } from './helpers/skill-fixture';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
@@ -72,14 +73,17 @@ function mkEvalRoot(suffix: string, includeOverlay: boolean): string {
|
||||
throw new Error(`gen-skill-docs failed: ${result.stderr}`);
|
||||
}
|
||||
|
||||
// Install per-skill SKILL.md files for Skill tool discovery.
|
||||
// Install per-skill SKILL.md files for Skill tool discovery. Routing only
|
||||
// reads the frontmatter (name + description), so install frontmatter + the
|
||||
// first ~30 body lines instead of the full 1000-1900-line files
|
||||
// (CLAUDE.md: "E2E test fixtures: extract, don't copy").
|
||||
const skillsDir = path.join(tmp, '.claude', 'skills');
|
||||
for (const skill of INSTALLED_SKILLS) {
|
||||
const src = path.join(ROOT, skill, 'SKILL.md');
|
||||
if (!fs.existsSync(src)) continue;
|
||||
const destDir = path.join(skillsDir, skill);
|
||||
fs.mkdirSync(destDir, { recursive: true });
|
||||
fs.copyFileSync(src, path.join(destDir, 'SKILL.md'));
|
||||
fs.writeFileSync(path.join(destDir, 'SKILL.md'), extractSkillHead(src));
|
||||
}
|
||||
|
||||
// Extract the opus-4-7 model-overlay content from the checked-in file
|
||||
|
||||
@@ -99,7 +99,21 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
|
||||
// outcome === 'asked' would let a silent-bypass run that reaches
|
||||
// plan_ready (isPlanReadyVisible also matches common prose) sail
|
||||
// through — the exact regression this test exists to catch.
|
||||
expect(obs.scopeGateQuestionObserved ?? false).toBe(true);
|
||||
//
|
||||
// Throw WITH the evidence tail instead of a bare expect: this member
|
||||
// (plan-design-review especially) intermittently fails ONLY this
|
||||
// check on unchanged code (PR #2593 rounds 3/11/rerun, passing
|
||||
// rounds 5/6), and a bare Expected-true/Received-false in CI logs is
|
||||
// undiagnosable — we can't tell a detector-sensitivity miss (render
|
||||
// shape scrolled/rephrased) from a real silent bypass without seeing
|
||||
// what the screen held.
|
||||
if (!(obs.scopeGateQuestionObserved ?? false)) {
|
||||
throw new Error(
|
||||
`scope-gate question NOT observed (${skillName}): outcome=${obs.outcome}\n` +
|
||||
`elapsed: ${obs.elapsedMs}ms\n` +
|
||||
`--- evidence (last 2KB visible) ---\n${obs.evidence}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}, 360_000);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
import { expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
describeIfSelected, testConcurrentIfSelected,
|
||||
logCost, recordE2E,
|
||||
createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { extractSkillSections, RETRO_E2E_SECTIONS } from './helpers/skill-fixture';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
const evalCollector = createEvalCollector('e2e-retro');
|
||||
|
||||
// --- Retro base branch detection smoke test ---
|
||||
|
||||
describeIfSelected('Base branch detection', ['retro-base-branch'], () => {
|
||||
let baseBranchDir: string;
|
||||
const run = (cmd: string, args: string[], cwd: string) =>
|
||||
spawnSync(cmd, args, { cwd, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
beforeAll(() => {
|
||||
baseBranchDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-basebranch-'));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(baseBranchDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
testConcurrentIfSelected('retro-base-branch', async () => {
|
||||
const dir = path.join(baseBranchDir, 'retro-base');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
|
||||
// Create git repo with commit history
|
||||
run('git', ['init'], dir);
|
||||
run('git', ['config', 'user.email', 'dev@example.com'], dir);
|
||||
run('git', ['config', 'user.name', 'Dev'], dir);
|
||||
|
||||
fs.writeFileSync(path.join(dir, 'app.ts'), 'console.log("hello");\n');
|
||||
run('git', ['add', 'app.ts'], dir);
|
||||
run('git', ['commit', '-m', 'feat: initial app', '--date', '2026-03-14T09:00:00'], dir);
|
||||
|
||||
fs.writeFileSync(path.join(dir, 'auth.ts'), 'export function login() {}\n');
|
||||
run('git', ['add', 'auth.ts'], dir);
|
||||
run('git', ['commit', '-m', 'feat: add auth', '--date', '2026-03-15T10:00:00'], dir);
|
||||
|
||||
fs.writeFileSync(path.join(dir, 'test.ts'), 'test("it works", () => {});\n');
|
||||
run('git', ['add', 'test.ts'], dir);
|
||||
run('git', ['commit', '-m', 'test: add tests', '--date', '2026-03-16T11:00:00'], dir);
|
||||
|
||||
// Retro skill — extract the repo-scoped retro flow only (drops the shared
|
||||
// preamble + global/compare modes; CLAUDE.md: "extract, don't copy").
|
||||
fs.mkdirSync(path.join(dir, 'retro'), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(dir, 'retro', 'SKILL.md'),
|
||||
extractSkillSections(path.join(ROOT, 'retro'), RETRO_E2E_SECTIONS),
|
||||
);
|
||||
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read retro/SKILL.md for instructions on how to run a retrospective.
|
||||
|
||||
IMPORTANT: Follow the "Detect default branch" step first. Since there is no remote, gh will fail — fall back to main.
|
||||
Then use the detected branch name for all git queries.
|
||||
|
||||
Run /retro for the last 7 days of this git repo. Skip any AskUserQuestion calls — this is non-interactive.
|
||||
This is a local-only repo so use the local branch (main) instead of origin/main for all git log commands.
|
||||
|
||||
Write your retrospective to ${dir}/retro-output.md`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 25,
|
||||
// 360s, not 240s: same runner-contention class as review-dashboard-via.
|
||||
// /retro is a long multi-step flow — a clean pass measured 225s and the
|
||||
// next CI run timed out at the 240s line (exitReason "timeout", 3/3
|
||||
// attempts). Outer bun timeout below rises to 480s for headroom.
|
||||
timeout: 360_000,
|
||||
testName: 'retro-base-branch',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/retro base-branch', result);
|
||||
// The report is the work product: a run that exits max-turns without
|
||||
// writing it is a FAIL, not a pass — otherwise this test cannot detect
|
||||
// the most basic regression (the skill stops producing its report).
|
||||
const retroPath = path.join(dir, 'retro-output.md');
|
||||
const wroteReport = fs.existsSync(retroPath);
|
||||
recordE2E(evalCollector, '/retro default branch detection', 'Base branch detection', result, {
|
||||
passed: ['success', 'error_max_turns'].includes(result.exitReason) && wroteReport,
|
||||
});
|
||||
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
||||
expect(wroteReport).toBe(true);
|
||||
const content = fs.readFileSync(retroPath, 'utf-8');
|
||||
expect(content.length).toBeGreaterThan(100);
|
||||
}, 480_000);
|
||||
});
|
||||
|
||||
// --- Retro E2E ---
|
||||
|
||||
describeIfSelected('Retro E2E', ['retro'], () => {
|
||||
let retroDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
retroDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-retro-'));
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: retroDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
// Create a git repo with varied commit history
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'dev@example.com']);
|
||||
run('git', ['config', 'user.name', 'Dev']);
|
||||
|
||||
// Day 1 commits
|
||||
fs.writeFileSync(path.join(retroDir, 'app.ts'), 'console.log("hello");\n');
|
||||
run('git', ['add', 'app.ts']);
|
||||
run('git', ['commit', '-m', 'feat: initial app setup', '--date', '2026-03-10T09:00:00']);
|
||||
|
||||
fs.writeFileSync(path.join(retroDir, 'auth.ts'), 'export function login() {}\n');
|
||||
run('git', ['add', 'auth.ts']);
|
||||
run('git', ['commit', '-m', 'feat: add auth module', '--date', '2026-03-10T11:00:00']);
|
||||
|
||||
// Day 2 commits
|
||||
fs.writeFileSync(path.join(retroDir, 'app.ts'), 'import { login } from "./auth";\nconsole.log("hello");\nlogin();\n');
|
||||
run('git', ['add', 'app.ts']);
|
||||
run('git', ['commit', '-m', 'fix: wire up auth to app', '--date', '2026-03-11T10:00:00']);
|
||||
|
||||
fs.writeFileSync(path.join(retroDir, 'test.ts'), 'import { test } from "bun:test";\ntest("login", () => {});\n');
|
||||
run('git', ['add', 'test.ts']);
|
||||
run('git', ['commit', '-m', 'test: add login test', '--date', '2026-03-11T14:00:00']);
|
||||
|
||||
// Day 3 commits
|
||||
fs.writeFileSync(path.join(retroDir, 'api.ts'), 'export function getUsers() { return []; }\n');
|
||||
run('git', ['add', 'api.ts']);
|
||||
run('git', ['commit', '-m', 'feat: add users API endpoint', '--date', '2026-03-12T09:30:00']);
|
||||
|
||||
fs.writeFileSync(path.join(retroDir, 'README.md'), '# My App\nA test application.\n');
|
||||
run('git', ['add', 'README.md']);
|
||||
run('git', ['commit', '-m', 'docs: add README', '--date', '2026-03-12T16:00:00']);
|
||||
|
||||
// Retro skill — extracted repo-scoped flow, not the full 1820-line file.
|
||||
fs.mkdirSync(path.join(retroDir, 'retro'), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(retroDir, 'retro', 'SKILL.md'),
|
||||
extractSkillSections(path.join(ROOT, 'retro'), RETRO_E2E_SECTIONS),
|
||||
);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(retroDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
testConcurrentIfSelected('retro', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read retro/SKILL.md for instructions on how to run a retrospective.
|
||||
|
||||
Run /retro for the last 7 days of this git repo. Skip any AskUserQuestion calls — this is non-interactive.
|
||||
Write your retrospective report to ${retroDir}/retro-output.md
|
||||
|
||||
Analyze the git history and produce the narrative report as described in the SKILL.md.`,
|
||||
workingDirectory: retroDir,
|
||||
maxTurns: 30,
|
||||
timeout: 300_000,
|
||||
testName: 'retro',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
});
|
||||
|
||||
logCost('/retro', result);
|
||||
// Accept error_max_turns (retro does many git commands to analyze
|
||||
// history) — but only WITH the report on disk. The report is the work
|
||||
// product; max-turns with nothing written is a fail.
|
||||
const retroPath = path.join(retroDir, 'retro-output.md');
|
||||
const wroteReport = fs.existsSync(retroPath);
|
||||
recordE2E(evalCollector, '/retro', 'Retro E2E', result, {
|
||||
passed: ['success', 'error_max_turns'].includes(result.exitReason) && wroteReport,
|
||||
});
|
||||
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
||||
expect(wroteReport).toBe(true);
|
||||
const retro = fs.readFileSync(retroPath, 'utf-8');
|
||||
expect(retro.length).toBeGreaterThan(100);
|
||||
}, 420_000);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
afterAll(async () => {
|
||||
await finalizeEvalCollector(evalCollector);
|
||||
});
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
ROOT, runId, describeIfSelected, testConcurrentIfSelected,
|
||||
logCost, recordE2E, createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { extractSkillSections, REVIEW_ARMY_E2E_SECTIONS } from './helpers/skill-fixture';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
@@ -22,9 +23,15 @@ function setupRepo(prefix: string): { dir: string; run: (cmd: string, args: stri
|
||||
return { dir, run };
|
||||
}
|
||||
|
||||
// Helper: copy review skill files to test dir
|
||||
// Helper: stage review skill files in the test dir. The SKILL.md fixture is
|
||||
// EXTRACTED (CLAUDE.md: "E2E test fixtures: extract, don't copy") — core
|
||||
// review workflow + Step 1.5 (Plan Completion Audit) + Step 4.5 (Review Army
|
||||
// dispatch: quality score, JSON schema, consensus, Red Team).
|
||||
function copyReviewFiles(dir: string) {
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'SKILL.md'), path.join(dir, 'review-SKILL.md'));
|
||||
fs.writeFileSync(
|
||||
path.join(dir, 'review-SKILL.md'),
|
||||
extractSkillSections(path.join(ROOT, 'review'), REVIEW_ARMY_E2E_SECTIONS),
|
||||
);
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(dir, 'review-checklist.md'));
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(dir, 'review-greptile-triage.md'));
|
||||
// Copy specialist checklists
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
import { expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
describeIfSelected, testConcurrentIfSelected,
|
||||
logCost, recordE2E,
|
||||
createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
const evalCollector = createEvalCollector('e2e-review-attribution');
|
||||
|
||||
// --- Base branch detection smoke tests ---
|
||||
|
||||
describeIfSelected('Base branch detection', ['review-base-branch', 'ship-base-branch'], () => {
|
||||
let baseBranchDir: string;
|
||||
const run = (cmd: string, args: string[], cwd: string) =>
|
||||
spawnSync(cmd, args, { cwd, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
beforeAll(() => {
|
||||
baseBranchDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-basebranch-'));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(baseBranchDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
testConcurrentIfSelected('review-base-branch', async () => {
|
||||
const dir = path.join(baseBranchDir, 'review-base');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
|
||||
// Create git repo with a feature branch off main
|
||||
run('git', ['init'], dir);
|
||||
run('git', ['config', 'user.email', 'test@test.com'], dir);
|
||||
run('git', ['config', 'user.name', 'Test'], dir);
|
||||
|
||||
fs.writeFileSync(path.join(dir, 'app.rb'), '# clean base\nclass App\nend\n');
|
||||
run('git', ['add', 'app.rb'], dir);
|
||||
run('git', ['commit', '-m', 'initial commit'], dir);
|
||||
|
||||
// Create feature branch with a change
|
||||
run('git', ['checkout', '-b', 'feature/test-review'], dir);
|
||||
fs.writeFileSync(path.join(dir, 'app.rb'), '# clean base\nclass App\n def hello; "world"; end\nend\n');
|
||||
run('git', ['add', 'app.rb'], dir);
|
||||
run('git', ['commit', '-m', 'feat: add hello method'], dir);
|
||||
|
||||
// Extract only Step 0 (base branch detection) + minimal review instructions
|
||||
// Full SKILL.md is ~1500 lines — copying it causes the agent to spend all turns reading
|
||||
const full = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
|
||||
const step0Start = full.indexOf('## Step 0: Detect platform and base branch');
|
||||
const step1Start = full.indexOf('## Step 1: Check branch');
|
||||
const step1End = full.indexOf('---', step1Start + 10);
|
||||
const extracted = full.slice(step0Start, step1End > step1Start ? step1End : step1Start + 500);
|
||||
fs.writeFileSync(path.join(dir, 'review-SKILL.md'), extracted);
|
||||
|
||||
const result = await runSkillTest({
|
||||
prompt: `You are in a git repo on a feature branch with changes.
|
||||
Read review-SKILL.md for the base branch detection instructions.
|
||||
|
||||
IMPORTANT: Follow Step 0 to detect the base branch. Since there is no remote, gh commands will fail — fall back to main.
|
||||
Then run git diff against the detected base branch and write a brief review.
|
||||
Write your findings to ${dir}/review-output.md`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 15,
|
||||
timeout: 90_000,
|
||||
testName: 'review-base-branch',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/review base-branch', result);
|
||||
recordE2E(evalCollector, '/review base branch detection', 'Base branch detection', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
|
||||
// Verify the review used "base branch" language (from Step 0)
|
||||
const toolOutputs = result.toolCalls.map(tc => tc.output || '').join('\n');
|
||||
const allOutput = (result.output || '') + toolOutputs;
|
||||
// The agent should have run git diff against main (the fallback)
|
||||
const usedGitDiff = result.toolCalls.some(tc => {
|
||||
if (tc.tool !== 'Bash') return false;
|
||||
const cmd = typeof tc.input === 'string' ? tc.input : tc.input?.command || JSON.stringify(tc.input);
|
||||
return cmd.includes('git diff');
|
||||
});
|
||||
expect(usedGitDiff).toBe(true);
|
||||
}, 120_000);
|
||||
|
||||
testConcurrentIfSelected('ship-base-branch', async () => {
|
||||
const dir = path.join(baseBranchDir, 'ship-base');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
|
||||
// Create git repo with feature branch
|
||||
run('git', ['init'], dir);
|
||||
run('git', ['config', 'user.email', 'test@test.com'], dir);
|
||||
run('git', ['config', 'user.name', 'Test'], dir);
|
||||
|
||||
fs.writeFileSync(path.join(dir, 'app.ts'), 'console.log("v1");\n');
|
||||
run('git', ['add', 'app.ts'], dir);
|
||||
run('git', ['commit', '-m', 'initial'], dir);
|
||||
|
||||
run('git', ['checkout', '-b', 'feature/ship-test'], dir);
|
||||
fs.writeFileSync(path.join(dir, 'app.ts'), 'console.log("v2");\n');
|
||||
run('git', ['add', 'app.ts'], dir);
|
||||
run('git', ['commit', '-m', 'feat: update to v2'], dir);
|
||||
|
||||
// Extract only Step 0 (base branch detection) from ship/SKILL.md
|
||||
// (copying the full 1900-line file causes agent context bloat and flaky timeouts)
|
||||
const fullShipSkill = fs.readFileSync(path.join(ROOT, 'ship', 'SKILL.md'), 'utf-8');
|
||||
const step0Start = fullShipSkill.indexOf('## Step 0: Detect platform and base branch');
|
||||
const step0End = fullShipSkill.indexOf('## Step 1: Pre-flight');
|
||||
const shipSection = fullShipSkill.slice(step0Start, step0End > step0Start ? step0End : undefined);
|
||||
fs.writeFileSync(path.join(dir, 'ship-SKILL.md'), shipSection);
|
||||
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read ship-SKILL.md. It contains Step 0 (Detect base branch) from the ship workflow.
|
||||
|
||||
Run the base branch detection. Since there is no remote, gh commands will fail — fall back to main.
|
||||
|
||||
Then run git diff and git log against the detected base branch.
|
||||
|
||||
Write a summary to ${dir}/ship-preflight.md including:
|
||||
- The detected base branch name
|
||||
- The current branch name
|
||||
- The diff stat against the base branch`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 18,
|
||||
timeout: 150_000,
|
||||
testName: 'ship-base-branch',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/ship base-branch', result);
|
||||
recordE2E(evalCollector, '/ship base branch detection', 'Base branch detection', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
|
||||
// Verify preflight output was written
|
||||
const preflightPath = path.join(dir, 'ship-preflight.md');
|
||||
if (fs.existsSync(preflightPath)) {
|
||||
const content = fs.readFileSync(preflightPath, 'utf-8');
|
||||
expect(content.length).toBeGreaterThan(20);
|
||||
// Should mention the branch name
|
||||
expect(content.toLowerCase()).toMatch(/main|base/);
|
||||
}
|
||||
|
||||
// Verify no destructive actions — no push, no PR creation
|
||||
// session-runner records tool inputs as OBJECTS ({command} for Bash) —
|
||||
// a typeof-string filter here matches nothing and the assertion can
|
||||
// never fail, even against a real `git push`.
|
||||
const destructiveTools = result.toolCalls.filter(tc => {
|
||||
if (tc.tool !== 'Bash') return false;
|
||||
const command = typeof tc.input === 'string'
|
||||
? tc.input
|
||||
: ((tc.input as { command?: string })?.command ?? JSON.stringify(tc.input ?? {}));
|
||||
return command.includes('git push') || command.includes('gh pr create');
|
||||
});
|
||||
expect(destructiveTools).toHaveLength(0);
|
||||
}, 180_000);
|
||||
});
|
||||
|
||||
// --- Review Dashboard Via Attribution E2E ---
|
||||
|
||||
describeIfSelected('Review Dashboard Via Attribution', ['review-dashboard-via'], () => {
|
||||
let dashDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
dashDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-dashboard-via-'));
|
||||
const run = (cmd: string, args: string[], cwd = dashDir) =>
|
||||
spawnSync(cmd, args, { cwd, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
// Create git repo with feature branch
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
fs.writeFileSync(path.join(dashDir, 'app.ts'), 'console.log("v1");\n');
|
||||
run('git', ['add', 'app.ts']);
|
||||
run('git', ['commit', '-m', 'initial']);
|
||||
|
||||
run('git', ['checkout', '-b', 'feature/dashboard-test']);
|
||||
fs.writeFileSync(path.join(dashDir, 'app.ts'), 'console.log("v2");\n');
|
||||
run('git', ['add', 'app.ts']);
|
||||
run('git', ['commit', '-m', 'feat: update']);
|
||||
|
||||
// Get HEAD commit for review entries
|
||||
const headResult = spawnSync('git', ['rev-parse', '--short', 'HEAD'], { cwd: dashDir, stdio: 'pipe' });
|
||||
const commit = headResult.stdout.toString().trim();
|
||||
|
||||
// Pre-populate review log with autoplan-sourced entries
|
||||
// gstack-review-read reads from ~/.gstack/projects/$SLUG/$BRANCH-reviews.jsonl
|
||||
// For the test, we'll write a mock gstack-review-read script that returns our test data
|
||||
const timestamp = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z');
|
||||
const reviewData = [
|
||||
`{"skill":"plan-eng-review","timestamp":"${timestamp}","status":"clean","unresolved":0,"critical_gaps":0,"issues_found":0,"mode":"FULL_REVIEW","via":"autoplan","commit":"${commit}"}`,
|
||||
`{"skill":"plan-ceo-review","timestamp":"${timestamp}","status":"clean","unresolved":0,"critical_gaps":0,"mode":"SELECTIVE_EXPANSION","via":"autoplan","commit":"${commit}"}`,
|
||||
`{"skill":"codex-plan-review","timestamp":"${timestamp}","status":"clean","source":"codex","commit":"${commit}"}`,
|
||||
].join('\n');
|
||||
|
||||
// Write a mock gstack-review-read that returns our test data
|
||||
const mockBinDir = path.join(dashDir, '.mock-bin');
|
||||
fs.mkdirSync(mockBinDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(mockBinDir, 'gstack-review-read'), [
|
||||
'#!/usr/bin/env bash',
|
||||
`echo '${reviewData.split('\n').join("'\necho '")}'`,
|
||||
'echo "---CONFIG---"',
|
||||
'echo "false"',
|
||||
'echo "---HEAD---"',
|
||||
`echo "${commit}"`,
|
||||
].join('\n'));
|
||||
fs.chmodSync(path.join(mockBinDir, 'gstack-review-read'), 0o755);
|
||||
|
||||
// Extract only the Review Readiness Dashboard section from ship/SKILL.md
|
||||
// (copying the full 1900-line file causes agent context bloat and timeouts)
|
||||
const fullSkill = fs.readFileSync(path.join(ROOT, 'ship', 'SKILL.md'), 'utf-8');
|
||||
const dashStart = fullSkill.indexOf('## Review Readiness Dashboard');
|
||||
const dashEnd = fullSkill.indexOf('\n---\n', dashStart);
|
||||
const dashSection = fullSkill.slice(dashStart, dashEnd > dashStart ? dashEnd : undefined);
|
||||
fs.writeFileSync(path.join(dashDir, 'ship-SKILL.md'), dashSection);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(dashDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
testConcurrentIfSelected('review-dashboard-via', async () => {
|
||||
const mockBinDir = path.join(dashDir, '.mock-bin');
|
||||
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read ship-SKILL.md. You only need to run the Review Readiness Dashboard section.
|
||||
|
||||
Instead of running ~/.claude/skills/gstack/bin/gstack-review-read, run this mock: ${mockBinDir}/gstack-review-read
|
||||
|
||||
Parse the output and display the dashboard table. Pay attention to:
|
||||
1. The "via" field in entries — show source attribution (e.g., "via /autoplan")
|
||||
2. The codex-plan-review entry — it should populate the Outside Voice row
|
||||
3. Since Eng Review IS clear, there should be NO gate blocking — just display the dashboard
|
||||
|
||||
Skip the preamble, lake intro, telemetry, and all other ship steps.
|
||||
Write the dashboard output to ${dashDir}/dashboard-output.md`,
|
||||
workingDirectory: dashDir,
|
||||
maxTurns: 12,
|
||||
// 360s, third ratchet of the same contention story: 180s deterministic
|
||||
// 0-turn timeouts (PR #2472) → 300s; then PR #2593 hit 302s timeouts
|
||||
// on attempt 2 in two consecutive runs while five sibling rounds
|
||||
// passed — the queue-behind-siblings startup tax under 40-way in-shard
|
||||
// concurrency is real and marginal at 300s. Same headroom its
|
||||
// contention-class sibling (retro-base-branch) already carries. Outer
|
||||
// bun timeout below rises to 480s to keep headroom over the inner.
|
||||
timeout: 360_000,
|
||||
testName: 'review-dashboard-via',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/ship dashboard-via', result);
|
||||
recordE2E(evalCollector, '/ship review dashboard via attribution', 'Dashboard via field', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
|
||||
// Check dashboard output for via attribution
|
||||
const dashPath = path.join(dashDir, 'dashboard-output.md');
|
||||
const allOutput = [
|
||||
result.output || '',
|
||||
...result.toolCalls.map(tc => tc.output || ''),
|
||||
].join('\n').toLowerCase();
|
||||
|
||||
// Verify via attribution appears somewhere (conversation or file)
|
||||
let dashContent = '';
|
||||
if (fs.existsSync(dashPath)) {
|
||||
dashContent = fs.readFileSync(dashPath, 'utf-8').toLowerCase();
|
||||
}
|
||||
const combined = allOutput + dashContent;
|
||||
|
||||
// Should mention autoplan attribution
|
||||
expect(combined).toMatch(/autoplan/);
|
||||
// Should show eng review as CLEAR (it has a clean entry)
|
||||
expect(combined).toMatch(/clear/i);
|
||||
// Should NOT contain AskUserQuestion gate (no blocking)
|
||||
const gateQuestions = result.toolCalls.filter(tc =>
|
||||
tc.tool === 'mcp__conductor__AskUserQuestion' ||
|
||||
(tc.tool === 'AskUserQuestion')
|
||||
);
|
||||
// Ship dashboard should not gate when eng review is clear
|
||||
expect(gateQuestions).toHaveLength(0);
|
||||
}, 480_000);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
afterAll(async () => {
|
||||
await finalizeEvalCollector(evalCollector);
|
||||
});
|
||||
+23
-414
@@ -6,6 +6,7 @@ import {
|
||||
copyDirSync, setupBrowseShims, logCost, recordE2E,
|
||||
createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { extractSkillSections, REVIEW_E2E_SECTIONS } from './helpers/skill-fixture';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
@@ -41,8 +42,12 @@ describeIfSelected('Review skill E2E', ['review-sql-injection'], () => {
|
||||
run('git', ['add', 'user_controller.rb']);
|
||||
run('git', ['commit', '-m', 'add user controller']);
|
||||
|
||||
// Copy review skill files
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'SKILL.md'), path.join(reviewDir, 'review-SKILL.md'));
|
||||
// Review skill files — extract only the core review workflow sections
|
||||
// (CLAUDE.md: "E2E test fixtures: extract, don't copy").
|
||||
fs.writeFileSync(
|
||||
path.join(reviewDir, 'review-SKILL.md'),
|
||||
extractSkillSections(path.join(ROOT, 'review'), REVIEW_E2E_SECTIONS),
|
||||
);
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(reviewDir, 'review-checklist.md'));
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(reviewDir, 'review-greptile-triage.md'));
|
||||
});
|
||||
@@ -115,8 +120,11 @@ describeIfSelected('Review enum completeness E2E', ['review-enum-completeness'],
|
||||
run('git', ['add', 'order.rb']);
|
||||
run('git', ['commit', '-m', 'add returned status']);
|
||||
|
||||
// Copy review skill files
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'SKILL.md'), path.join(enumDir, 'review-SKILL.md'));
|
||||
// Review skill files — extracted sections, not the full 1870-line file.
|
||||
fs.writeFileSync(
|
||||
path.join(enumDir, 'review-SKILL.md'),
|
||||
extractSkillSections(path.join(ROOT, 'review'), REVIEW_E2E_SECTIONS),
|
||||
);
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(enumDir, 'review-checklist.md'));
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(enumDir, 'review-greptile-triage.md'));
|
||||
});
|
||||
@@ -189,8 +197,13 @@ describeIfSelected('Review design lite E2E', ['review-design-lite'], () => {
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'add landing page']);
|
||||
|
||||
// Copy review skill files
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'SKILL.md'), path.join(designDir, 'review-SKILL.md'));
|
||||
// Review skill files — extracted sections, not the full 1870-line file.
|
||||
// The design checks come from review-design-checklist.md (copied whole,
|
||||
// it is a 134-line checklist, not a generated SKILL.md).
|
||||
fs.writeFileSync(
|
||||
path.join(designDir, 'review-SKILL.md'),
|
||||
extractSkillSections(path.join(ROOT, 'review'), REVIEW_E2E_SECTIONS),
|
||||
);
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'checklist.md'), path.join(designDir, 'review-checklist.md'));
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'design-checklist.md'), path.join(designDir, 'review-design-checklist.md'));
|
||||
fs.copyFileSync(path.join(ROOT, 'review', 'greptile-triage.md'), path.join(designDir, 'review-greptile-triage.md'));
|
||||
@@ -252,414 +265,10 @@ Important: The design checklist should catch issues like blacklisted fonts, smal
|
||||
}, 300_000);
|
||||
});
|
||||
|
||||
// --- Base branch detection smoke tests ---
|
||||
|
||||
describeIfSelected('Base branch detection', ['review-base-branch', 'ship-base-branch', 'retro-base-branch'], () => {
|
||||
let baseBranchDir: string;
|
||||
const run = (cmd: string, args: string[], cwd: string) =>
|
||||
spawnSync(cmd, args, { cwd, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
beforeAll(() => {
|
||||
baseBranchDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-basebranch-'));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(baseBranchDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
testConcurrentIfSelected('review-base-branch', async () => {
|
||||
const dir = path.join(baseBranchDir, 'review-base');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
|
||||
// Create git repo with a feature branch off main
|
||||
run('git', ['init'], dir);
|
||||
run('git', ['config', 'user.email', 'test@test.com'], dir);
|
||||
run('git', ['config', 'user.name', 'Test'], dir);
|
||||
|
||||
fs.writeFileSync(path.join(dir, 'app.rb'), '# clean base\nclass App\nend\n');
|
||||
run('git', ['add', 'app.rb'], dir);
|
||||
run('git', ['commit', '-m', 'initial commit'], dir);
|
||||
|
||||
// Create feature branch with a change
|
||||
run('git', ['checkout', '-b', 'feature/test-review'], dir);
|
||||
fs.writeFileSync(path.join(dir, 'app.rb'), '# clean base\nclass App\n def hello; "world"; end\nend\n');
|
||||
run('git', ['add', 'app.rb'], dir);
|
||||
run('git', ['commit', '-m', 'feat: add hello method'], dir);
|
||||
|
||||
// Extract only Step 0 (base branch detection) + minimal review instructions
|
||||
// Full SKILL.md is ~1500 lines — copying it causes the agent to spend all turns reading
|
||||
const full = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
|
||||
const step0Start = full.indexOf('## Step 0: Detect platform and base branch');
|
||||
const step1Start = full.indexOf('## Step 1: Check branch');
|
||||
const step1End = full.indexOf('---', step1Start + 10);
|
||||
const extracted = full.slice(step0Start, step1End > step1Start ? step1End : step1Start + 500);
|
||||
fs.writeFileSync(path.join(dir, 'review-SKILL.md'), extracted);
|
||||
|
||||
const result = await runSkillTest({
|
||||
prompt: `You are in a git repo on a feature branch with changes.
|
||||
Read review-SKILL.md for the base branch detection instructions.
|
||||
|
||||
IMPORTANT: Follow Step 0 to detect the base branch. Since there is no remote, gh commands will fail — fall back to main.
|
||||
Then run git diff against the detected base branch and write a brief review.
|
||||
Write your findings to ${dir}/review-output.md`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 15,
|
||||
timeout: 90_000,
|
||||
testName: 'review-base-branch',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/review base-branch', result);
|
||||
recordE2E(evalCollector, '/review base branch detection', 'Base branch detection', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
|
||||
// Verify the review used "base branch" language (from Step 0)
|
||||
const toolOutputs = result.toolCalls.map(tc => tc.output || '').join('\n');
|
||||
const allOutput = (result.output || '') + toolOutputs;
|
||||
// The agent should have run git diff against main (the fallback)
|
||||
const usedGitDiff = result.toolCalls.some(tc => {
|
||||
if (tc.tool !== 'Bash') return false;
|
||||
const cmd = typeof tc.input === 'string' ? tc.input : tc.input?.command || JSON.stringify(tc.input);
|
||||
return cmd.includes('git diff');
|
||||
});
|
||||
expect(usedGitDiff).toBe(true);
|
||||
}, 120_000);
|
||||
|
||||
testConcurrentIfSelected('ship-base-branch', async () => {
|
||||
const dir = path.join(baseBranchDir, 'ship-base');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
|
||||
// Create git repo with feature branch
|
||||
run('git', ['init'], dir);
|
||||
run('git', ['config', 'user.email', 'test@test.com'], dir);
|
||||
run('git', ['config', 'user.name', 'Test'], dir);
|
||||
|
||||
fs.writeFileSync(path.join(dir, 'app.ts'), 'console.log("v1");\n');
|
||||
run('git', ['add', 'app.ts'], dir);
|
||||
run('git', ['commit', '-m', 'initial'], dir);
|
||||
|
||||
run('git', ['checkout', '-b', 'feature/ship-test'], dir);
|
||||
fs.writeFileSync(path.join(dir, 'app.ts'), 'console.log("v2");\n');
|
||||
run('git', ['add', 'app.ts'], dir);
|
||||
run('git', ['commit', '-m', 'feat: update to v2'], dir);
|
||||
|
||||
// Extract only Step 0 (base branch detection) from ship/SKILL.md
|
||||
// (copying the full 1900-line file causes agent context bloat and flaky timeouts)
|
||||
const fullShipSkill = fs.readFileSync(path.join(ROOT, 'ship', 'SKILL.md'), 'utf-8');
|
||||
const step0Start = fullShipSkill.indexOf('## Step 0: Detect platform and base branch');
|
||||
const step0End = fullShipSkill.indexOf('## Step 1: Pre-flight');
|
||||
const shipSection = fullShipSkill.slice(step0Start, step0End > step0Start ? step0End : undefined);
|
||||
fs.writeFileSync(path.join(dir, 'ship-SKILL.md'), shipSection);
|
||||
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read ship-SKILL.md. It contains Step 0 (Detect base branch) from the ship workflow.
|
||||
|
||||
Run the base branch detection. Since there is no remote, gh commands will fail — fall back to main.
|
||||
|
||||
Then run git diff and git log against the detected base branch.
|
||||
|
||||
Write a summary to ${dir}/ship-preflight.md including:
|
||||
- The detected base branch name
|
||||
- The current branch name
|
||||
- The diff stat against the base branch`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 18,
|
||||
timeout: 150_000,
|
||||
testName: 'ship-base-branch',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/ship base-branch', result);
|
||||
recordE2E(evalCollector, '/ship base branch detection', 'Base branch detection', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
|
||||
// Verify preflight output was written
|
||||
const preflightPath = path.join(dir, 'ship-preflight.md');
|
||||
if (fs.existsSync(preflightPath)) {
|
||||
const content = fs.readFileSync(preflightPath, 'utf-8');
|
||||
expect(content.length).toBeGreaterThan(20);
|
||||
// Should mention the branch name
|
||||
expect(content.toLowerCase()).toMatch(/main|base/);
|
||||
}
|
||||
|
||||
// Verify no destructive actions — no push, no PR creation
|
||||
const destructiveTools = result.toolCalls.filter(tc =>
|
||||
tc.tool === 'Bash' && typeof tc.input === 'string' &&
|
||||
(tc.input.includes('git push') || tc.input.includes('gh pr create'))
|
||||
);
|
||||
expect(destructiveTools).toHaveLength(0);
|
||||
}, 180_000);
|
||||
|
||||
testConcurrentIfSelected('retro-base-branch', async () => {
|
||||
const dir = path.join(baseBranchDir, 'retro-base');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
|
||||
// Create git repo with commit history
|
||||
run('git', ['init'], dir);
|
||||
run('git', ['config', 'user.email', 'dev@example.com'], dir);
|
||||
run('git', ['config', 'user.name', 'Dev'], dir);
|
||||
|
||||
fs.writeFileSync(path.join(dir, 'app.ts'), 'console.log("hello");\n');
|
||||
run('git', ['add', 'app.ts'], dir);
|
||||
run('git', ['commit', '-m', 'feat: initial app', '--date', '2026-03-14T09:00:00'], dir);
|
||||
|
||||
fs.writeFileSync(path.join(dir, 'auth.ts'), 'export function login() {}\n');
|
||||
run('git', ['add', 'auth.ts'], dir);
|
||||
run('git', ['commit', '-m', 'feat: add auth', '--date', '2026-03-15T10:00:00'], dir);
|
||||
|
||||
fs.writeFileSync(path.join(dir, 'test.ts'), 'test("it works", () => {});\n');
|
||||
run('git', ['add', 'test.ts'], dir);
|
||||
run('git', ['commit', '-m', 'test: add tests', '--date', '2026-03-16T11:00:00'], dir);
|
||||
|
||||
// Copy retro skill
|
||||
fs.mkdirSync(path.join(dir, 'retro'), { recursive: true });
|
||||
fs.copyFileSync(path.join(ROOT, 'retro', 'SKILL.md'), path.join(dir, 'retro', 'SKILL.md'));
|
||||
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read retro/SKILL.md for instructions on how to run a retrospective.
|
||||
|
||||
IMPORTANT: Follow the "Detect default branch" step first. Since there is no remote, gh will fail — fall back to main.
|
||||
Then use the detected branch name for all git queries.
|
||||
|
||||
Run /retro for the last 7 days of this git repo. Skip any AskUserQuestion calls — this is non-interactive.
|
||||
This is a local-only repo so use the local branch (main) instead of origin/main for all git log commands.
|
||||
|
||||
Write your retrospective to ${dir}/retro-output.md`,
|
||||
workingDirectory: dir,
|
||||
maxTurns: 25,
|
||||
// 360s, not 240s: same runner-contention class as review-dashboard-via.
|
||||
// /retro is a long multi-step flow — a clean pass measured 225s and the
|
||||
// next CI run timed out at the 240s line (exitReason "timeout", 3/3
|
||||
// attempts). Outer bun timeout below rises to 480s for headroom.
|
||||
timeout: 360_000,
|
||||
testName: 'retro-base-branch',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/retro base-branch', result);
|
||||
recordE2E(evalCollector, '/retro default branch detection', 'Base branch detection', result, {
|
||||
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
||||
});
|
||||
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
||||
|
||||
// Verify retro output was produced
|
||||
const retroPath = path.join(dir, 'retro-output.md');
|
||||
if (fs.existsSync(retroPath)) {
|
||||
const content = fs.readFileSync(retroPath, 'utf-8');
|
||||
expect(content.length).toBeGreaterThan(100);
|
||||
}
|
||||
}, 480_000);
|
||||
});
|
||||
|
||||
// --- Retro E2E ---
|
||||
|
||||
describeIfSelected('Retro E2E', ['retro'], () => {
|
||||
let retroDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
retroDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-retro-'));
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: retroDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
// Create a git repo with varied commit history
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'dev@example.com']);
|
||||
run('git', ['config', 'user.name', 'Dev']);
|
||||
|
||||
// Day 1 commits
|
||||
fs.writeFileSync(path.join(retroDir, 'app.ts'), 'console.log("hello");\n');
|
||||
run('git', ['add', 'app.ts']);
|
||||
run('git', ['commit', '-m', 'feat: initial app setup', '--date', '2026-03-10T09:00:00']);
|
||||
|
||||
fs.writeFileSync(path.join(retroDir, 'auth.ts'), 'export function login() {}\n');
|
||||
run('git', ['add', 'auth.ts']);
|
||||
run('git', ['commit', '-m', 'feat: add auth module', '--date', '2026-03-10T11:00:00']);
|
||||
|
||||
// Day 2 commits
|
||||
fs.writeFileSync(path.join(retroDir, 'app.ts'), 'import { login } from "./auth";\nconsole.log("hello");\nlogin();\n');
|
||||
run('git', ['add', 'app.ts']);
|
||||
run('git', ['commit', '-m', 'fix: wire up auth to app', '--date', '2026-03-11T10:00:00']);
|
||||
|
||||
fs.writeFileSync(path.join(retroDir, 'test.ts'), 'import { test } from "bun:test";\ntest("login", () => {});\n');
|
||||
run('git', ['add', 'test.ts']);
|
||||
run('git', ['commit', '-m', 'test: add login test', '--date', '2026-03-11T14:00:00']);
|
||||
|
||||
// Day 3 commits
|
||||
fs.writeFileSync(path.join(retroDir, 'api.ts'), 'export function getUsers() { return []; }\n');
|
||||
run('git', ['add', 'api.ts']);
|
||||
run('git', ['commit', '-m', 'feat: add users API endpoint', '--date', '2026-03-12T09:30:00']);
|
||||
|
||||
fs.writeFileSync(path.join(retroDir, 'README.md'), '# My App\nA test application.\n');
|
||||
run('git', ['add', 'README.md']);
|
||||
run('git', ['commit', '-m', 'docs: add README', '--date', '2026-03-12T16:00:00']);
|
||||
|
||||
// Copy retro skill
|
||||
fs.mkdirSync(path.join(retroDir, 'retro'), { recursive: true });
|
||||
fs.copyFileSync(
|
||||
path.join(ROOT, 'retro', 'SKILL.md'),
|
||||
path.join(retroDir, 'retro', 'SKILL.md'),
|
||||
);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(retroDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
testConcurrentIfSelected('retro', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read retro/SKILL.md for instructions on how to run a retrospective.
|
||||
|
||||
Run /retro for the last 7 days of this git repo. Skip any AskUserQuestion calls — this is non-interactive.
|
||||
Write your retrospective report to ${retroDir}/retro-output.md
|
||||
|
||||
Analyze the git history and produce the narrative report as described in the SKILL.md.`,
|
||||
workingDirectory: retroDir,
|
||||
maxTurns: 30,
|
||||
timeout: 300_000,
|
||||
testName: 'retro',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
});
|
||||
|
||||
logCost('/retro', result);
|
||||
recordE2E(evalCollector, '/retro', 'Retro E2E', result, {
|
||||
passed: ['success', 'error_max_turns'].includes(result.exitReason),
|
||||
});
|
||||
// Accept error_max_turns — retro does many git commands to analyze history
|
||||
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
||||
|
||||
// Verify the retro was written
|
||||
const retroPath = path.join(retroDir, 'retro-output.md');
|
||||
if (fs.existsSync(retroPath)) {
|
||||
const retro = fs.readFileSync(retroPath, 'utf-8');
|
||||
expect(retro.length).toBeGreaterThan(100);
|
||||
}
|
||||
}, 420_000);
|
||||
});
|
||||
|
||||
// --- Review Dashboard Via Attribution E2E ---
|
||||
|
||||
describeIfSelected('Review Dashboard Via Attribution', ['review-dashboard-via'], () => {
|
||||
let dashDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
dashDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-dashboard-via-'));
|
||||
const run = (cmd: string, args: string[], cwd = dashDir) =>
|
||||
spawnSync(cmd, args, { cwd, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
// Create git repo with feature branch
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
fs.writeFileSync(path.join(dashDir, 'app.ts'), 'console.log("v1");\n');
|
||||
run('git', ['add', 'app.ts']);
|
||||
run('git', ['commit', '-m', 'initial']);
|
||||
|
||||
run('git', ['checkout', '-b', 'feature/dashboard-test']);
|
||||
fs.writeFileSync(path.join(dashDir, 'app.ts'), 'console.log("v2");\n');
|
||||
run('git', ['add', 'app.ts']);
|
||||
run('git', ['commit', '-m', 'feat: update']);
|
||||
|
||||
// Get HEAD commit for review entries
|
||||
const headResult = spawnSync('git', ['rev-parse', '--short', 'HEAD'], { cwd: dashDir, stdio: 'pipe' });
|
||||
const commit = headResult.stdout.toString().trim();
|
||||
|
||||
// Pre-populate review log with autoplan-sourced entries
|
||||
// gstack-review-read reads from ~/.gstack/projects/$SLUG/$BRANCH-reviews.jsonl
|
||||
// For the test, we'll write a mock gstack-review-read script that returns our test data
|
||||
const timestamp = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z');
|
||||
const reviewData = [
|
||||
`{"skill":"plan-eng-review","timestamp":"${timestamp}","status":"clean","unresolved":0,"critical_gaps":0,"issues_found":0,"mode":"FULL_REVIEW","via":"autoplan","commit":"${commit}"}`,
|
||||
`{"skill":"plan-ceo-review","timestamp":"${timestamp}","status":"clean","unresolved":0,"critical_gaps":0,"mode":"SELECTIVE_EXPANSION","via":"autoplan","commit":"${commit}"}`,
|
||||
`{"skill":"codex-plan-review","timestamp":"${timestamp}","status":"clean","source":"codex","commit":"${commit}"}`,
|
||||
].join('\n');
|
||||
|
||||
// Write a mock gstack-review-read that returns our test data
|
||||
const mockBinDir = path.join(dashDir, '.mock-bin');
|
||||
fs.mkdirSync(mockBinDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(mockBinDir, 'gstack-review-read'), [
|
||||
'#!/usr/bin/env bash',
|
||||
`echo '${reviewData.split('\n').join("'\necho '")}'`,
|
||||
'echo "---CONFIG---"',
|
||||
'echo "false"',
|
||||
'echo "---HEAD---"',
|
||||
`echo "${commit}"`,
|
||||
].join('\n'));
|
||||
fs.chmodSync(path.join(mockBinDir, 'gstack-review-read'), 0o755);
|
||||
|
||||
// Extract only the Review Readiness Dashboard section from ship/SKILL.md
|
||||
// (copying the full 1900-line file causes agent context bloat and timeouts)
|
||||
const fullSkill = fs.readFileSync(path.join(ROOT, 'ship', 'SKILL.md'), 'utf-8');
|
||||
const dashStart = fullSkill.indexOf('## Review Readiness Dashboard');
|
||||
const dashEnd = fullSkill.indexOf('\n---\n', dashStart);
|
||||
const dashSection = fullSkill.slice(dashStart, dashEnd > dashStart ? dashEnd : undefined);
|
||||
fs.writeFileSync(path.join(dashDir, 'ship-SKILL.md'), dashSection);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(dashDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
testConcurrentIfSelected('review-dashboard-via', async () => {
|
||||
const mockBinDir = path.join(dashDir, '.mock-bin');
|
||||
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read ship-SKILL.md. You only need to run the Review Readiness Dashboard section.
|
||||
|
||||
Instead of running ~/.claude/skills/gstack/bin/gstack-review-read, run this mock: ${mockBinDir}/gstack-review-read
|
||||
|
||||
Parse the output and display the dashboard table. Pay attention to:
|
||||
1. The "via" field in entries — show source attribution (e.g., "via /autoplan")
|
||||
2. The codex-plan-review entry — it should populate the Outside Voice row
|
||||
3. Since Eng Review IS clear, there should be NO gate blocking — just display the dashboard
|
||||
|
||||
Skip the preamble, lake intro, telemetry, and all other ship steps.
|
||||
Write the dashboard output to ${dashDir}/dashboard-output.md`,
|
||||
workingDirectory: dashDir,
|
||||
maxTurns: 12,
|
||||
// 300s, not 180s: on a saturated CI runner this file's concurrent
|
||||
// sessions queue behind each other and session STARTUP can eat the
|
||||
// whole budget — observed as deterministic timeout at 0 turns/$0.00
|
||||
// for exactly 180s across 3 attempts (PR #2472 CI + its baseline),
|
||||
// while the 240s-budget tests in the same job passed. Outer bun
|
||||
// timeout below rises to 360s to keep headroom over the inner budget.
|
||||
timeout: 300_000,
|
||||
testName: 'review-dashboard-via',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/ship dashboard-via', result);
|
||||
recordE2E(evalCollector, '/ship review dashboard via attribution', 'Dashboard via field', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
|
||||
// Check dashboard output for via attribution
|
||||
const dashPath = path.join(dashDir, 'dashboard-output.md');
|
||||
const allOutput = [
|
||||
result.output || '',
|
||||
...result.toolCalls.map(tc => tc.output || ''),
|
||||
].join('\n').toLowerCase();
|
||||
|
||||
// Verify via attribution appears somewhere (conversation or file)
|
||||
let dashContent = '';
|
||||
if (fs.existsSync(dashPath)) {
|
||||
dashContent = fs.readFileSync(dashPath, 'utf-8').toLowerCase();
|
||||
}
|
||||
const combined = allOutput + dashContent;
|
||||
|
||||
// Should mention autoplan attribution
|
||||
expect(combined).toMatch(/autoplan/);
|
||||
// Should show eng review as CLEAR (it has a clean entry)
|
||||
expect(combined).toMatch(/clear/i);
|
||||
// Should NOT contain AskUserQuestion gate (no blocking)
|
||||
const gateQuestions = result.toolCalls.filter(tc =>
|
||||
tc.tool === 'mcp__conductor__AskUserQuestion' ||
|
||||
(tc.tool === 'AskUserQuestion')
|
||||
);
|
||||
// Ship dashboard should not gate when eng review is clear
|
||||
expect(gateQuestions).toHaveLength(0);
|
||||
}, 360_000);
|
||||
});
|
||||
// Base branch detection tests for review/ship + the Review Dashboard Via
|
||||
// Attribution describe live in test/skill-e2e-review-attribution.test.ts.
|
||||
// Retro tests (retro, retro-base-branch) live in test/skill-e2e-retro.test.ts.
|
||||
// Split so CI's per-file matrix can run them in parallel.
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
afterAll(async () => {
|
||||
|
||||
@@ -11,11 +11,11 @@
|
||||
* 4. Does NOT append a duplicate CHANGELOG [0.0.2] entry
|
||||
* 5. Does NOT create a new "chore: bump version" commit
|
||||
*
|
||||
* Why real-PTY: the existing ship-idempotency test in skill-e2e.test.ts
|
||||
* uses the SDK harness with a synthetic prompt asking the agent to "run
|
||||
* ONLY the idempotency checks." This test exercises the actual /ship
|
||||
* skill end-to-end against a real git fixture so a regression that
|
||||
* silently re-bumps despite the check passing would be caught.
|
||||
* Why real-PTY: the old SDK-harness ship-idempotency variant (removed in
|
||||
* v1.64.1.0 as redundant with this test) used a synthetic prompt asking
|
||||
* the agent to "run ONLY the idempotency checks." This test exercises the
|
||||
* actual /ship skill end-to-end against a real git fixture so a regression
|
||||
* that silently re-bumps despite the check passing would be caught.
|
||||
*
|
||||
* Plan-mode framing: we run /ship in plan mode so the agent cannot push,
|
||||
* commit, or open PRs. The Step 12 idempotency check is read-only
|
||||
|
||||
@@ -34,6 +34,7 @@ import {
|
||||
setupBrowseShims, copyDirSync, logCost, recordE2E,
|
||||
createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { extractSkillBody } from './helpers/skill-fixture';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
@@ -77,12 +78,15 @@ function setupSkillifyWorkdir(suffix: string, installSkills: string[] = ['scrape
|
||||
|
||||
setupBrowseShims(workDir);
|
||||
|
||||
// Install requested skills.
|
||||
// Install requested skills. The tests exercise the full /scrape + /skillify
|
||||
// flows (all 11 skillify steps, D1-D3 contracts), so keep the whole
|
||||
// skill-specific body — but drop the ~780-line shared preamble the tests
|
||||
// never touch (CLAUDE.md: "E2E test fixtures: extract, don't copy").
|
||||
const skillsDir = path.join(workDir, '.claude', 'skills');
|
||||
for (const skill of installSkills) {
|
||||
const destDir = path.join(skillsDir, skill);
|
||||
fs.mkdirSync(destDir, { recursive: true });
|
||||
fs.copyFileSync(path.join(ROOT, skill, 'SKILL.md'), path.join(destDir, 'SKILL.md'));
|
||||
fs.writeFileSync(path.join(destDir, 'SKILL.md'), extractSkillBody(path.join(ROOT, skill)));
|
||||
}
|
||||
|
||||
// bin/ scripts — preamble references several of these.
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
/**
|
||||
* /ship test-failure ownership triage E2E.
|
||||
*
|
||||
* Rehomed VERBATIM from the pre-split monolith (test/skill-e2e.test.ts,
|
||||
* deleted on this branch): the monolith's filename never matched the paid
|
||||
* glob (`test/skill-e2e-*.test.ts` — note the hyphen), so this GATE-tier
|
||||
* test (`ship-triage` in E2E_TIERS) silently never executed after the
|
||||
* v1.56 split.
|
||||
*
|
||||
* DRIFT WARNING (attribution for the first paid run after rehoming): the
|
||||
* prompt references "Test Failure Ownership Triage (Steps T1-T4)" — no
|
||||
* such section exists in the current generated ship/SKILL.md (the skill
|
||||
* drifted while this test was a zombie). The body is copied faithfully
|
||||
* (no behavioral edits), so a failure here indicts the drift, not the
|
||||
* move. The only change vs the monolith body: the staged ship/SKILL.md is
|
||||
* extracted via test/helpers/skill-fixture.ts (extractSkillBody — full
|
||||
* skill-specific body, shared preamble dropped) per CLAUDE.md
|
||||
* "E2E test fixtures: extract, don't copy".
|
||||
*/
|
||||
|
||||
import { test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, runId,
|
||||
describeIfSelected,
|
||||
copyDirSync, logCost, recordE2E,
|
||||
createEvalCollector, finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { extractSkillBody } from './helpers/skill-fixture';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
||||
const evalCollector = createEvalCollector('e2e-triage');
|
||||
|
||||
// --- Triage E2E ---
|
||||
|
||||
describeIfSelected('Test Failure Triage E2E', ['ship-triage'], () => {
|
||||
let triageDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
triageDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-triage-'));
|
||||
|
||||
// Copy ship skill files, then replace the SKILL.md with the extracted
|
||||
// skill body (extract, don't copy).
|
||||
copyDirSync(path.join(ROOT, 'ship'), path.join(triageDir, 'ship'));
|
||||
fs.writeFileSync(
|
||||
path.join(triageDir, 'ship', 'SKILL.md'),
|
||||
extractSkillBody(path.join(ROOT, 'ship')),
|
||||
);
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: triageDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
// Init git repo
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
// Create a project with a pre-existing test failure on main
|
||||
fs.writeFileSync(path.join(triageDir, 'package.json'), JSON.stringify({
|
||||
name: 'triage-test-app',
|
||||
version: '1.0.0',
|
||||
scripts: { test: 'node test/run.js' },
|
||||
}, null, 2));
|
||||
|
||||
fs.mkdirSync(path.join(triageDir, 'src'), { recursive: true });
|
||||
fs.mkdirSync(path.join(triageDir, 'test'), { recursive: true });
|
||||
|
||||
// Source with a bug that exists on main (pre-existing)
|
||||
fs.writeFileSync(path.join(triageDir, 'src', 'math.js'), `
|
||||
module.exports = {
|
||||
add: (a, b) => a + b,
|
||||
divide: (a, b) => a / b, // BUG: no zero-division check (pre-existing)
|
||||
};
|
||||
`);
|
||||
|
||||
// Test file that catches the pre-existing bug
|
||||
fs.writeFileSync(path.join(triageDir, 'test', 'math.test.js'), `
|
||||
const { add, divide } = require('../src/math');
|
||||
|
||||
// This test passes
|
||||
if (add(2, 3) !== 5) { console.error('FAIL: add(2,3) should be 5'); process.exit(1); }
|
||||
console.log('PASS: add');
|
||||
|
||||
// This test FAILS — pre-existing bug (divide by zero returns Infinity, not an error)
|
||||
try {
|
||||
const result = divide(10, 0);
|
||||
if (result === Infinity) { console.error('FAIL: divide(10,0) should throw, got Infinity'); process.exit(1); }
|
||||
} catch(e) {
|
||||
console.log('PASS: divide zero check');
|
||||
}
|
||||
`);
|
||||
|
||||
// Test runner — each test in a subprocess so one failure doesn't kill the other
|
||||
fs.writeFileSync(path.join(triageDir, 'test', 'run.js'), `
|
||||
const { execSync } = require('child_process');
|
||||
const path = require('path');
|
||||
let failures = 0;
|
||||
for (const f of ['math.test.js', 'string.test.js']) {
|
||||
try {
|
||||
execSync('node ' + path.join(__dirname, f), { stdio: 'inherit' });
|
||||
} catch (e) {
|
||||
failures++;
|
||||
}
|
||||
}
|
||||
if (failures > 0) process.exit(1);
|
||||
`);
|
||||
|
||||
// Commit on main with the pre-existing bug
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'initial: math utils with tests']);
|
||||
|
||||
// Create feature branch
|
||||
run('git', ['checkout', '-b', 'feature/string-utils']);
|
||||
|
||||
// Add new code with a new bug (in-branch)
|
||||
fs.writeFileSync(path.join(triageDir, 'src', 'string.js'), `
|
||||
module.exports = {
|
||||
capitalize: (s) => s.charAt(0).toUpperCase() + s.slice(1),
|
||||
reverse: (s) => s.split('').reverse().join(''),
|
||||
truncate: (s, len) => s.substring(0, len), // BUG: no null check (in-branch)
|
||||
};
|
||||
`);
|
||||
|
||||
// Add test that catches the in-branch bug
|
||||
fs.writeFileSync(path.join(triageDir, 'test', 'string.test.js'), `
|
||||
const { capitalize, reverse, truncate } = require('../src/string');
|
||||
|
||||
if (capitalize('hello') !== 'Hello') { console.error('FAIL: capitalize'); process.exit(1); }
|
||||
console.log('PASS: capitalize');
|
||||
|
||||
if (reverse('abc') !== 'cba') { console.error('FAIL: reverse'); process.exit(1); }
|
||||
console.log('PASS: reverse');
|
||||
|
||||
// This test FAILS — in-branch bug (null input causes TypeError)
|
||||
try {
|
||||
truncate(null, 5);
|
||||
console.log('PASS: truncate null');
|
||||
} catch(e) {
|
||||
console.error('FAIL: truncate(null, 5) threw: ' + e.message);
|
||||
process.exit(1);
|
||||
}
|
||||
`);
|
||||
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'feat: add string utilities']);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(triageDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
test('/ship triage correctly classifies in-branch vs pre-existing failures', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read the file ship/SKILL.md for the ship workflow instructions.
|
||||
|
||||
You are on the feature/string-utils branch. The base branch is main.
|
||||
This is a test project — there is no remote, no PR to create.
|
||||
|
||||
Run the tests first:
|
||||
\`\`\`bash
|
||||
cd ${triageDir} && node test/run.js
|
||||
\`\`\`
|
||||
|
||||
The tests will fail. Now run ONLY the Test Failure Ownership Triage (Steps T1-T4) from the ship workflow.
|
||||
|
||||
For each failing test, classify it as:
|
||||
- **In-branch**: caused by changes on this branch (feature/string-utils)
|
||||
- **Pre-existing**: existed before this branch (present on main)
|
||||
|
||||
Use git diff origin/main...HEAD (or git diff main...HEAD since there's no remote) to determine which files changed on this branch.
|
||||
|
||||
Output your classification for each failure clearly, labeling each as "IN-BRANCH" or "PRE-EXISTING" with your reasoning.
|
||||
|
||||
This is a solo repo (REPO_MODE=solo). For pre-existing failures, recommend fixing now.`,
|
||||
workingDirectory: triageDir,
|
||||
maxTurns: 20,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep'],
|
||||
timeout: 180_000,
|
||||
testName: 'ship-triage',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('/ship triage', result);
|
||||
|
||||
const output = result.output || '';
|
||||
const outputLower = output.toLowerCase();
|
||||
|
||||
// The triage should identify the string/truncate failure as in-branch
|
||||
const hasInBranch = outputLower.includes('in-branch') || outputLower.includes('in branch') || outputLower.includes('introduced');
|
||||
// The triage should identify the math/divide failure as pre-existing
|
||||
const hasPreExisting = outputLower.includes('pre-existing') || outputLower.includes('pre existing') || outputLower.includes('existed before');
|
||||
|
||||
console.log(`Output identifies IN-BRANCH failures: ${hasInBranch}`);
|
||||
console.log(`Output identifies PRE-EXISTING failures: ${hasPreExisting}`);
|
||||
|
||||
// Check that the string/truncate bug is classified as in-branch
|
||||
const mentionsTruncate = outputLower.includes('truncate') || outputLower.includes('string');
|
||||
const mentionsDivide = outputLower.includes('divide') || outputLower.includes('math');
|
||||
|
||||
console.log(`Mentions truncate/string (in-branch bug): ${mentionsTruncate}`);
|
||||
console.log(`Mentions divide/math (pre-existing bug): ${mentionsDivide}`);
|
||||
|
||||
// Verify BOTH failure classes are exercised (not just detected):
|
||||
// The test runner must have actually run both test files
|
||||
const ranMathTest = output.includes('math.test') || output.includes('FAIL: divide');
|
||||
const ranStringTest = output.includes('string.test') || output.includes('FAIL: truncate');
|
||||
console.log(`Ran math test file (pre-existing failure): ${ranMathTest}`);
|
||||
console.log(`Ran string test file (in-branch failure): ${ranStringTest}`);
|
||||
|
||||
recordE2E(evalCollector, '/ship triage', 'Test Failure Triage E2E', result, {
|
||||
passed: result.exitReason === 'success' && hasInBranch && hasPreExisting,
|
||||
has_in_branch_classification: hasInBranch,
|
||||
has_pre_existing_classification: hasPreExisting,
|
||||
mentions_truncate: mentionsTruncate,
|
||||
mentions_divide: mentionsDivide,
|
||||
ran_both_test_files: ranMathTest && ranStringTest,
|
||||
});
|
||||
|
||||
expect(result.exitReason).toBe('success');
|
||||
// Must classify at least one failure as in-branch AND one as pre-existing
|
||||
expect(hasInBranch).toBe(true);
|
||||
expect(hasPreExisting).toBe(true);
|
||||
// Must mention the specific bugs
|
||||
expect(mentionsTruncate).toBe(true);
|
||||
expect(mentionsDivide).toBe(true);
|
||||
// Must have actually run both test files (exercises both failure classes)
|
||||
expect(ranMathTest).toBe(true);
|
||||
expect(ranStringTest).toBe(true);
|
||||
}, 240_000);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
afterAll(async () => {
|
||||
await finalizeEvalCollector(evalCollector);
|
||||
});
|
||||
@@ -77,7 +77,13 @@ IMPORTANT:
|
||||
workingDirectory: docReleaseDir,
|
||||
maxTurns: 30,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 180_000,
|
||||
// 300s, not 180s: a 30-turn multi-step doc workflow under 40-way
|
||||
// in-shard CI concurrency timed out at exactly 180s on its final
|
||||
// attempt twice on PR #2593 (rounds 4 and 13) while passing four
|
||||
// other rounds — marginal at 180s, same contention story as
|
||||
// review-dashboard-via and retro-base-branch. Outer bun timeout
|
||||
// rises to 360s for headroom.
|
||||
timeout: 300_000,
|
||||
testName: 'document-release',
|
||||
runId,
|
||||
});
|
||||
@@ -114,7 +120,7 @@ IMPORTANT:
|
||||
} else {
|
||||
console.warn('README was NOT updated — agent may not have found the feature');
|
||||
}
|
||||
}, 240_000);
|
||||
}, 360_000);
|
||||
});
|
||||
|
||||
// --- Ship workflow with local bare remote ---
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,249 @@
|
||||
/**
|
||||
* Unit + pin tests for test/helpers/skill-fixture.ts (free tier, no EVALS).
|
||||
*
|
||||
* Two layers:
|
||||
* 1. Semantics against a synthetic SKILL.md: frontmatter always included,
|
||||
* sections concatenated in caller order, missing section throws with the
|
||||
* section name, fenced `## ` template headings do not split sections,
|
||||
* body extraction drops exactly the shared preamble block, head
|
||||
* extraction truncates the body.
|
||||
* 2. Pins against the REAL generated SKILL.md files: every exported section
|
||||
* list extracts cleanly from the skill it targets, and the body/head
|
||||
* helpers work for every skill the E2E fixtures feed through them. A
|
||||
* heading rename in gen-skill-docs fails HERE (free, <1s) instead of
|
||||
* mid-flight in a paid E2E run.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import {
|
||||
extractSkillSections,
|
||||
extractSkillBody,
|
||||
extractSkillHead,
|
||||
REVIEW_E2E_SECTIONS,
|
||||
REVIEW_ARMY_E2E_SECTIONS,
|
||||
RETRO_E2E_SECTIONS,
|
||||
CODEX_REVIEW_E2E_SECTIONS,
|
||||
} from './helpers/skill-fixture';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
// ─── Synthetic fixture ──────────────────────────────────────────────────────
|
||||
|
||||
const SYNTHETIC_SKILL = `---
|
||||
name: fixture-test
|
||||
description: synthetic skill for skill-fixture unit tests
|
||||
---
|
||||
Intro line before any section.
|
||||
|
||||
## When to invoke this skill
|
||||
Invoke text.
|
||||
|
||||
## Preamble (run first)
|
||||
preamble junk that fixtures must drop
|
||||
|
||||
## AskUserQuestion Format
|
||||
more shared-preamble junk
|
||||
|
||||
## Plan Status Footer
|
||||
footer junk, last shared-preamble section
|
||||
|
||||
## Step 1 — Do the thing
|
||||
step one body
|
||||
\`\`\`markdown
|
||||
## Embedded Template Heading
|
||||
template content inside a fence
|
||||
\`\`\`
|
||||
step one continues after the fence
|
||||
|
||||
## Step 2 — Other
|
||||
step two body
|
||||
`;
|
||||
|
||||
let tmpDir: string;
|
||||
let skillDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-fixture-test-'));
|
||||
skillDir = path.join(tmpDir, 'fixture-test');
|
||||
fs.mkdirSync(skillDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(skillDir, 'SKILL.md'), SYNTHETIC_SKILL);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
describe('extractSkillSections (synthetic)', () => {
|
||||
test('always includes frontmatter and concatenates sections in caller order', () => {
|
||||
const out = extractSkillSections(skillDir, ['Step 2 — Other', 'Step 1 — Do the thing']);
|
||||
expect(out.startsWith('---\nname: fixture-test')).toBe(true);
|
||||
expect(out).toContain('## Step 1 — Do the thing');
|
||||
expect(out).toContain('## Step 2 — Other');
|
||||
// Caller order preserved: Step 2 requested first, so it appears first.
|
||||
expect(out.indexOf('## Step 2 — Other')).toBeLessThan(out.indexOf('## Step 1 — Do the thing'));
|
||||
// Unrequested sections are dropped.
|
||||
expect(out).not.toContain('preamble junk');
|
||||
expect(out).not.toContain('Intro line before any section');
|
||||
});
|
||||
|
||||
test('fenced ## headings do not terminate a section', () => {
|
||||
const out = extractSkillSections(skillDir, ['Step 1 — Do the thing']);
|
||||
expect(out).toContain('template content inside a fence');
|
||||
expect(out).toContain('step one continues after the fence');
|
||||
expect(out).not.toContain('step two body');
|
||||
});
|
||||
|
||||
test('missing section throws with the section name and the file path', () => {
|
||||
expect(() => extractSkillSections(skillDir, ['Step 99 — Renamed'])).toThrow(/Step 99 — Renamed/);
|
||||
expect(() => extractSkillSections(skillDir, ['Step 99 — Renamed'])).toThrow(/SKILL\.md/);
|
||||
});
|
||||
|
||||
test('a fenced heading is not findable as a section', () => {
|
||||
expect(() => extractSkillSections(skillDir, ['Embedded Template Heading'])).toThrow(/not found/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractSkillBody (synthetic)', () => {
|
||||
test('keeps frontmatter + intro + full body, drops the shared preamble block', () => {
|
||||
const out = extractSkillBody(skillDir);
|
||||
expect(out.startsWith('---\nname: fixture-test')).toBe(true);
|
||||
expect(out).toContain('Intro line before any section');
|
||||
expect(out).toContain('## When to invoke this skill');
|
||||
expect(out).toContain('## Step 1 — Do the thing');
|
||||
expect(out).toContain('template content inside a fence');
|
||||
expect(out).toContain('## Step 2 — Other');
|
||||
expect(out).not.toContain('preamble junk');
|
||||
expect(out).not.toContain('shared-preamble junk');
|
||||
expect(out).not.toContain('footer junk');
|
||||
});
|
||||
|
||||
test('throws when the preamble markers are missing', () => {
|
||||
const bare = path.join(tmpDir, 'bare');
|
||||
fs.mkdirSync(bare, { recursive: true });
|
||||
fs.writeFileSync(path.join(bare, 'SKILL.md'), '---\nname: bare\n---\n## Only Section\nbody\n');
|
||||
expect(() => extractSkillBody(bare)).toThrow(/Preamble \(run first\)/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractSkillHead (synthetic)', () => {
|
||||
test('keeps frontmatter + first N body lines only', () => {
|
||||
const out = extractSkillHead(skillDir, 3);
|
||||
expect(out.startsWith('---\nname: fixture-test')).toBe(true);
|
||||
expect(out).toContain('Intro line before any section');
|
||||
expect(out).toContain('## When to invoke this skill');
|
||||
expect(out).not.toContain('## Step 1 — Do the thing');
|
||||
expect(out).toContain('body truncated by test/helpers/skill-fixture.ts');
|
||||
});
|
||||
});
|
||||
|
||||
describe('error polarity', () => {
|
||||
test('missing SKILL.md throws (never writes an empty fixture)', () => {
|
||||
expect(() => extractSkillSections(path.join(tmpDir, 'nope'), ['x'])).toThrow(/no SKILL\.md/);
|
||||
});
|
||||
|
||||
test('file without frontmatter throws', () => {
|
||||
const nofm = path.join(tmpDir, 'nofm');
|
||||
fs.mkdirSync(nofm, { recursive: true });
|
||||
fs.writeFileSync(path.join(nofm, 'SKILL.md'), '# no frontmatter\n## Section\n');
|
||||
expect(() => extractSkillHead(nofm)).toThrow(/frontmatter/);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Pins against the real generated SKILL.md files ─────────────────────────
|
||||
// These turn "someone renamed a section in gen-skill-docs" into a FREE test
|
||||
// failure instead of a paid E2E setup throw.
|
||||
|
||||
describe('real-skill pins: section lists used by E2E fixtures', () => {
|
||||
test('REVIEW_E2E_SECTIONS extracts from review/SKILL.md', () => {
|
||||
const out = extractSkillSections(path.join(ROOT, 'review'), REVIEW_E2E_SECTIONS);
|
||||
expect(out).toContain('## Step 4: Critical pass (core review)');
|
||||
expect(out).toContain('## Important Rules');
|
||||
// Drops the shared preamble and the untested workflow tail.
|
||||
expect(out).not.toContain('## Telemetry (run last)');
|
||||
expect(out).not.toContain('## Step 5: Fix-First Review');
|
||||
// Meaningfully smaller than the source.
|
||||
const full = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
|
||||
expect(out.length).toBeLessThan(full.length * 0.5);
|
||||
});
|
||||
|
||||
test('REVIEW_ARMY_E2E_SECTIONS extracts from review/SKILL.md', () => {
|
||||
const out = extractSkillSections(path.join(ROOT, 'review'), REVIEW_ARMY_E2E_SECTIONS);
|
||||
// The army tests reference the Plan Completion Audit (inside Step 1.5)
|
||||
// and the Step 4.5 merge machinery (quality score, JSON schema, consensus).
|
||||
expect(out).toContain('PLAN COMPLETION AUDIT');
|
||||
expect(out).toContain('## Step 4.5: Review Army — Specialist Dispatch');
|
||||
expect(out).toContain('quality_score');
|
||||
expect(out).toContain('MULTI-SPECIALIST CONFIRMED');
|
||||
expect(out).not.toContain('## Telemetry (run last)');
|
||||
});
|
||||
|
||||
test('RETRO_E2E_SECTIONS extracts from retro/SKILL.md', () => {
|
||||
const out = extractSkillSections(path.join(ROOT, 'retro'), RETRO_E2E_SECTIONS);
|
||||
// Steps 0.5-14 live under Prior Learnings / Capture Learnings.
|
||||
expect(out).toContain('### Step 1: Gather Raw Data');
|
||||
expect(out).toContain('### Step 14: Write the Narrative');
|
||||
expect(out).toContain('## Engineering Retro: [date range]');
|
||||
expect(out).not.toContain('## Global Retrospective Mode');
|
||||
expect(out).not.toContain('## Telemetry (run last)');
|
||||
});
|
||||
|
||||
test('CODEX_REVIEW_E2E_SECTIONS extracts from the Codex host variant when present', () => {
|
||||
const codexReview = path.join(ROOT, '.agents', 'skills', 'gstack-review');
|
||||
if (!fs.existsSync(path.join(codexReview, 'SKILL.md'))) return; // gitignored artifact, absent in fresh checkouts
|
||||
const out = extractSkillSections(codexReview, CODEX_REVIEW_E2E_SECTIONS);
|
||||
expect(out).toContain('## Step 4: Critical pass (core review)');
|
||||
expect(out).not.toContain('## Telemetry (run last)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('real-skill pins: body/head extraction used by E2E fixtures', () => {
|
||||
// scrape/skillify/context-*: skill-e2e-skillify + skill-e2e-context-skills.
|
||||
// review/plan-eng-review/ship: skill-e2e-coverage-audit + skill-e2e-triage.
|
||||
const BODY_EXTRACTED_SKILLS = [
|
||||
'scrape', 'skillify', 'context-save', 'context-restore',
|
||||
'review', 'plan-eng-review', 'ship',
|
||||
];
|
||||
|
||||
for (const skill of BODY_EXTRACTED_SKILLS) {
|
||||
test(`extractSkillBody(${skill}) drops the shared preamble, keeps the flow`, () => {
|
||||
const out = extractSkillBody(path.join(ROOT, skill));
|
||||
expect(out).not.toContain('## Preamble (run first)');
|
||||
expect(out).not.toContain('## Telemetry (run last)');
|
||||
const full = fs.readFileSync(path.join(ROOT, skill, 'SKILL.md'), 'utf-8');
|
||||
expect(out.length).toBeLessThan(full.length * 0.75);
|
||||
expect(out.length).toBeGreaterThan(500);
|
||||
});
|
||||
}
|
||||
|
||||
test('body extraction keeps the sections the skillify/context E2E tests assert on', () => {
|
||||
expect(extractSkillBody(path.join(ROOT, 'skillify'))).toContain('## Step 1 — Provenance guard (D1)');
|
||||
expect(extractSkillBody(path.join(ROOT, 'scrape'))).toContain('## Step 4 — Prototype phase');
|
||||
expect(extractSkillBody(path.join(ROOT, 'context-save'))).toContain('## List flow');
|
||||
expect(extractSkillBody(path.join(ROOT, 'context-restore'))).toContain('## If no saved contexts exist');
|
||||
});
|
||||
|
||||
// The union of skills installed by the routing + opus-47 discovery fixtures.
|
||||
const HEAD_EXTRACTED_SKILLS = [
|
||||
'', 'qa', 'qa-only', 'ship', 'review', 'plan-ceo-review', 'plan-eng-review',
|
||||
'plan-design-review', 'design-review', 'design-consultation', 'retro',
|
||||
'document-release', 'investigate', 'office-hours', 'browse',
|
||||
'setup-browser-cookies', 'gstack-upgrade', 'humanizer',
|
||||
];
|
||||
|
||||
test('extractSkillHead works for every discovery-fixture skill', () => {
|
||||
for (const skill of HEAD_EXTRACTED_SKILLS) {
|
||||
const src = path.join(ROOT, skill, 'SKILL.md');
|
||||
if (!fs.existsSync(src)) continue; // mirrors the fixtures' existsSync guard
|
||||
const out = extractSkillHead(src);
|
||||
expect(out.startsWith('---\n')).toBe(true);
|
||||
expect(out).toContain('description:');
|
||||
// Frontmatter length varies (allowed-tools + triggers); the invariant
|
||||
// is "frontmatter + 30 body lines + marker", never the full body.
|
||||
const fullLines = fs.readFileSync(src, 'utf-8').split('\n').length;
|
||||
expect(out.split('\n').length).toBeLessThan(Math.min(150, fullLines));
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -4,6 +4,7 @@ import type { SkillTestResult } from './helpers/session-runner';
|
||||
import { EvalCollector } from './helpers/eval-store';
|
||||
import type { EvalTestEntry } from './helpers/eval-store';
|
||||
import { selectTests, detectBaseBranch, getChangedFiles, E2E_TOUCHFILES, E2E_TIERS, GLOBAL_TOUCHFILES } from './helpers/touchfiles';
|
||||
import { extractSkillHead } from './helpers/skill-fixture';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
@@ -59,10 +60,14 @@ if (evalsEnabled && process.env.EVALS_TIER) {
|
||||
|
||||
// --- Helper functions ---
|
||||
|
||||
/** Copy all SKILL.md files for auto-discovery.
|
||||
/** Install SKILL.md fixtures for auto-discovery.
|
||||
* Installs to project-level (.claude/skills/) only. Writing to the user's
|
||||
* ~/.claude/skills/ is unsafe: it may contain symlinks from the real gstack
|
||||
* install that point to different worktrees or dangling targets. */
|
||||
* install that point to different worktrees or dangling targets.
|
||||
*
|
||||
* ROUTING tests only read each skill's frontmatter (name + description) to
|
||||
* pick a skill, so install frontmatter + the first ~30 body lines instead
|
||||
* of ~20 full 1000-1900-line files (CLAUDE.md: "extract, don't copy"). */
|
||||
function installSkills(tmpDir: string) {
|
||||
const skillDirs = [
|
||||
'', // root gstack SKILL.md
|
||||
@@ -81,7 +86,7 @@ function installSkills(tmpDir: string) {
|
||||
const skillName = skill || 'gstack';
|
||||
const destDir = path.join(targetBase, skillName);
|
||||
fs.mkdirSync(destDir, { recursive: true });
|
||||
fs.copyFileSync(srcPath, path.join(destDir, 'SKILL.md'));
|
||||
fs.writeFileSync(path.join(destDir, 'SKILL.md'), extractSkillHead(srcPath));
|
||||
}
|
||||
|
||||
// Write a CLAUDE.md with explicit routing instructions.
|
||||
|
||||
@@ -31,7 +31,8 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { captureBaseline, type ParityBaseline } from './helpers/capture-parity-baseline';
|
||||
import { execSync } from 'child_process';
|
||||
import { captureBaseline, extractDescription, type ParityBaseline } from './helpers/capture-parity-baseline';
|
||||
import { logBudgetOverride } from './helpers/budget-override';
|
||||
import { CARVED_SKILLS } from './helpers/carve-guards';
|
||||
|
||||
@@ -227,11 +228,31 @@ describe('SKILL.md size budget regression (gate, free)', () => {
|
||||
});
|
||||
|
||||
test('catalog token estimate stays compressed (v1.45 target ≤ 7000)', () => {
|
||||
const current = captureBaseline({ repoRoot: REPO_ROOT });
|
||||
// Measure COMMITTED content (git show HEAD:), not the live tree. Under
|
||||
// the parallel free-suite runner, sibling workers regenerate real
|
||||
// SKILL.md files mid-run (gen-skill-docs regen tests), so the live-tree
|
||||
// estimate was a moving target: 4177 solo, 8356 and 8041 in two parallel
|
||||
// runs. A repo-budget ratchet measures the catalog that ships; CI always
|
||||
// checks the PR's committed tree anyway.
|
||||
const trackedPaths = execSync('git ls-files -- "*/SKILL.md"', { cwd: REPO_ROOT, encoding: 'utf-8' })
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.filter((p) => p.split('/').length === 2);
|
||||
let descriptionBytes = 0;
|
||||
for (const rel of trackedPaths) {
|
||||
const committed = execSync(`git show HEAD:${JSON.stringify(rel)}`, {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
});
|
||||
descriptionBytes += Buffer.byteLength(extractDescription(committed), 'utf-8');
|
||||
}
|
||||
const catalogTokens = Math.round(descriptionBytes / 4);
|
||||
const trackedCount = trackedPaths.length;
|
||||
const v145Target = 7000;
|
||||
if (current.estTotalCatalogTokens <= v145Target) {
|
||||
if (catalogTokens <= v145Target) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`[skill-size-budget] catalog OK: ~${current.estTotalCatalogTokens} tokens (target ≤${v145Target})`);
|
||||
console.log(`[skill-size-budget] catalog OK: ~${catalogTokens} tokens (target ≤${v145Target}, ${trackedCount} tracked skills)`);
|
||||
return;
|
||||
}
|
||||
const overrideReason = process.env.GSTACK_SIZE_BUDGET_OVERRIDE_REASON?.trim();
|
||||
@@ -239,12 +260,12 @@ describe('SKILL.md size budget regression (gate, free)', () => {
|
||||
logBudgetOverride({
|
||||
scope: 'skill-size-budget-catalog',
|
||||
reason: overrideReason,
|
||||
details: { target: v145Target, observed: current.estTotalCatalogTokens },
|
||||
details: { target: v145Target, observed: catalogTokens },
|
||||
});
|
||||
return;
|
||||
}
|
||||
throw new Error(
|
||||
`Catalog token estimate regressed past v1.45 target: ${current.estTotalCatalogTokens} tokens > ${v145Target}. ` +
|
||||
`Catalog token estimate regressed past v1.45 target: ${catalogTokens} tokens > ${v145Target}. ` +
|
||||
`T4 catalog trim should keep this under control. Override: set GSTACK_SIZE_BUDGET_OVERRIDE_REASON to allow.`,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -20,6 +20,17 @@ describe('/spec template/generated sync', () => {
|
||||
cwd: ROOT,
|
||||
encoding: 'utf-8',
|
||||
timeout: 120_000,
|
||||
// Scrubbed env: bun test runs a shard's files serially in ONE process,
|
||||
// so an earlier test's env mutations (GSTACK_*/GBRAIN_* detection vars)
|
||||
// leak into inherited process.env and change generator output — this
|
||||
// test failed in-suite while passing solo on an identical tree. The
|
||||
// generator's output must be a function of the templates, not of
|
||||
// whichever test ran before this one.
|
||||
env: {
|
||||
PATH: process.env.PATH ?? '',
|
||||
HOME: process.env.HOME ?? '',
|
||||
TMPDIR: process.env.TMPDIR ?? '',
|
||||
},
|
||||
});
|
||||
expect(res.status).toBe(0);
|
||||
|
||||
|
||||
+103
-1
@@ -8,7 +8,14 @@
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'bun:test';
|
||||
import { BunTestOutputClassifier, strictTestExitCode } from '../scripts/test-strict-output';
|
||||
import {
|
||||
BunTestOutputClassifier,
|
||||
installChildSignalForwarding,
|
||||
isTerminationRequested,
|
||||
strictTestExitCode,
|
||||
type TerminationSignalSource,
|
||||
type TerminationTimerApi,
|
||||
} from '../scripts/test-strict-output';
|
||||
|
||||
describe('strictTestExitCode', () => {
|
||||
it('trusts a clean zero exit when the expected file count ran', () => {
|
||||
@@ -58,4 +65,99 @@ describe('BunTestOutputClassifier', () => {
|
||||
// passes: 1 file ran, which is what was expected
|
||||
expect(strictTestExitCode(0, summary, 1)).toBe(0);
|
||||
});
|
||||
|
||||
// stdout and stderr are independent pipes: a chunk from one can land
|
||||
// between two halves of a line from the other. A single shared buffer
|
||||
// glues the fragments into garbled lines — a sheared (fail) line goes
|
||||
// uncounted (defeating the exit-0-with-failures backstop) and a sheared
|
||||
// summary reads as truncation. Per-origin buffers keep each stream whole.
|
||||
it('a stderr chunk arriving mid-stdout-line does not shear either line', () => {
|
||||
const c = new BunTestOutputClassifier();
|
||||
c.write('some stdout noise without a newline yet', 'stdout');
|
||||
c.write('[31m(fail) planted [0.10ms][0m\n', 'stderr');
|
||||
c.write(' ...rest of the stdout line\n', 'stdout');
|
||||
const summary = c.end();
|
||||
expect(summary.failedTests).toBe(1);
|
||||
});
|
||||
|
||||
it('a terminal summary split around a cross-stream chunk still counts', () => {
|
||||
const c = new BunTestOutputClassifier();
|
||||
c.write('Ran 4 tests acr', 'stdout');
|
||||
c.write('stderr diagnostics line\n', 'stderr');
|
||||
c.write('oss 2 files. [1.00s]\n', 'stdout');
|
||||
const summary = c.end();
|
||||
expect(summary.terminalFileCounts).toEqual([2]);
|
||||
expect(strictTestExitCode(0, summary, 2)).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('installChildSignalForwarding — cancellation terminates the RUN', () => {
|
||||
// Installing any SIGINT/SIGTERM listener suppresses Node's default
|
||||
// terminate-on-signal. Pre-fix, the forwarder killed the current child and
|
||||
// the parent LIVED ON — the paid worker pool kept launching API-burning
|
||||
// shards after Ctrl-C. The parent must schedule its own exit and expose
|
||||
// isTerminationRequested() so launch loops stop taking new work.
|
||||
type Handler = () => void;
|
||||
const makeFakes = () => {
|
||||
const listeners = new Map<string, Handler[]>();
|
||||
const source: TerminationSignalSource = {
|
||||
on: (event, listener) => {
|
||||
listeners.set(event, [...(listeners.get(event) ?? []), listener]);
|
||||
},
|
||||
off: (event, listener) => {
|
||||
listeners.set(event, (listeners.get(event) ?? []).filter((l) => l !== listener));
|
||||
},
|
||||
};
|
||||
const emit = (event: string) => (listeners.get(event) ?? []).forEach((l) => l());
|
||||
const scheduled: Array<{ callback: () => void; delayMs: number; cancelled: boolean }> = [];
|
||||
const timer: TerminationTimerApi = {
|
||||
schedule: (callback, delayMs) => {
|
||||
const handle = { callback, delayMs, cancelled: false };
|
||||
scheduled.push(handle);
|
||||
return handle;
|
||||
},
|
||||
cancel: (handle) => {
|
||||
(handle as { cancelled: boolean }).cancelled = true;
|
||||
},
|
||||
};
|
||||
const kills: string[] = [];
|
||||
const child = { kill: (sig?: unknown) => { kills.push(String(sig)); return true; } };
|
||||
const exits: number[] = [];
|
||||
return { source, emit, timer, scheduled, kills, child, exits, exit: (code: number) => { exits.push(code); } };
|
||||
};
|
||||
|
||||
it('first signal kills the child, marks termination, and schedules parent exit after the grace', () => {
|
||||
const f = makeFakes();
|
||||
installChildSignalForwarding(f.child, f.source, f.timer, 5_000, f.exit);
|
||||
expect(isTerminationRequested(f.source)).toBe(false);
|
||||
f.emit('SIGTERM');
|
||||
expect(f.kills).toEqual(['SIGTERM']);
|
||||
expect(isTerminationRequested(f.source)).toBe(true);
|
||||
// Two timers: child SIGKILL grace (5s) and parent exit (grace + 1s).
|
||||
const delays = f.scheduled.map((s) => s.delayMs);
|
||||
expect(delays).toContain(5_000);
|
||||
expect(delays).toContain(6_000);
|
||||
const parentExit = f.scheduled.find((s) => s.delayMs === 6_000)!;
|
||||
parentExit.callback();
|
||||
expect(f.exits).toEqual([143]);
|
||||
});
|
||||
|
||||
it('parent exit fires even when the shard disposes cleanly first', () => {
|
||||
const f = makeFakes();
|
||||
const forwarding = installChildSignalForwarding(f.child, f.source, f.timer, 5_000, f.exit);
|
||||
f.emit('SIGINT');
|
||||
forwarding.dispose();
|
||||
const parentExit = f.scheduled.find((s) => s.delayMs === 6_000)!;
|
||||
expect(parentExit.cancelled).toBe(false);
|
||||
parentExit.callback();
|
||||
expect(f.exits).toEqual([130]);
|
||||
});
|
||||
|
||||
it('one parent exit across many concurrent forwarders on the same source', () => {
|
||||
const f = makeFakes();
|
||||
installChildSignalForwarding(f.child, f.source, f.timer, 5_000, f.exit);
|
||||
installChildSignalForwarding({ kill: () => true }, f.source, f.timer, 5_000, f.exit);
|
||||
f.emit('SIGTERM');
|
||||
expect(f.scheduled.filter((s) => s.delayMs === 6_000).length).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,7 +9,19 @@ import {
|
||||
curateWindowsSafe,
|
||||
stableHash,
|
||||
assignFilesToShards,
|
||||
buildShardArgs,
|
||||
normalizeRelativePath,
|
||||
runFreeShard,
|
||||
FreeRunReporter,
|
||||
buildRunEpilogue,
|
||||
FREE_TEST_TIMEOUT_MS,
|
||||
DEFAULT_WALL_TIMEOUT_MS,
|
||||
PER_FILE_WALL_MS,
|
||||
wallTimeoutForShard,
|
||||
KNOWN_WINDOWS_INCOMPATIBLE,
|
||||
TEST_ROOTS,
|
||||
TREE_MUTATING,
|
||||
WORKER_HOSTILE,
|
||||
} from '../scripts/test-free-shards';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
@@ -106,12 +118,33 @@ describe('test-free-shards: sharding', () => {
|
||||
expect(stableHash('foo.test.ts')).not.toBe(stableHash('bar.test.ts'));
|
||||
});
|
||||
|
||||
test('assignFilesToShards distributes files into N non-empty shards', () => {
|
||||
test('assignFilesToShards partitions every file across exactly shardCount shards', () => {
|
||||
const files = ['a.test.ts', 'b.test.ts', 'c.test.ts', 'd.test.ts', 'e.test.ts'];
|
||||
const shards = assignFilesToShards(files, 3);
|
||||
const flattened = shards.flat();
|
||||
expect(flattened.sort()).toEqual([...files].sort());
|
||||
expect(shards.every((s) => s.length > 0)).toBe(true);
|
||||
expect(shards.length).toBe(3);
|
||||
expect(shards.flat().sort()).toEqual([...files].sort());
|
||||
});
|
||||
|
||||
test('empty shards are preserved so indices stay stable for a CI matrix', () => {
|
||||
// 2 files can never occupy 10 shards — the rest MUST be present and empty,
|
||||
// not filtered out (filtering renumbered every later shard by occupancy).
|
||||
const files = ['a.test.ts', 'b.test.ts'];
|
||||
const shards = assignFilesToShards(files, 10);
|
||||
expect(shards.length).toBe(10);
|
||||
expect(shards.flat().sort()).toEqual([...files].sort());
|
||||
expect(shards.some((s) => s.length === 0)).toBe(true);
|
||||
});
|
||||
|
||||
test("a file's shard index depends only on its own path — other files never renumber it", () => {
|
||||
const target = 'test/target.test.ts';
|
||||
const expected = stableHash(target) % 7;
|
||||
const alone = assignFilesToShards([target], 7);
|
||||
const crowded = assignFilesToShards(
|
||||
[target, 'test/a.test.ts', 'test/b.test.ts', 'test/c.test.ts', 'test/d.test.ts', 'browse/test/e.test.ts'],
|
||||
7,
|
||||
);
|
||||
expect(alone.findIndex((s) => s.includes(target))).toBe(expected);
|
||||
expect(crowded.findIndex((s) => s.includes(target))).toBe(expected);
|
||||
});
|
||||
|
||||
test('assignFilesToShards rejects invalid shard counts', () => {
|
||||
@@ -126,3 +159,409 @@ describe('test-free-shards: sharding', () => {
|
||||
expect(a).toEqual(b);
|
||||
});
|
||||
});
|
||||
|
||||
describe('test-free-shards: shard args', () => {
|
||||
test('resolves exact absolute selectors (no substring shard bleed) and pins the per-test timeout', () => {
|
||||
const args = buildShardArgs(['test/foo.test.ts'], { rootDir: ROOT });
|
||||
expect(args[0]).toBe('test');
|
||||
expect(args[1]).toBe(path.resolve(ROOT, 'test/foo.test.ts'));
|
||||
expect(args).toContain(`--timeout=${FREE_TEST_TIMEOUT_MS}`);
|
||||
expect(args).toContain('--max-concurrency=1');
|
||||
expect(args).not.toContain('--parallel');
|
||||
});
|
||||
|
||||
test('parallel mode swaps serial max-concurrency for --parallel', () => {
|
||||
const args = buildShardArgs(['test/foo.test.ts'], { rootDir: ROOT, parallel: true });
|
||||
expect(args).toContain('--parallel');
|
||||
expect(args).not.toContain('--max-concurrency=1');
|
||||
});
|
||||
|
||||
test('per-test timeout matches the 30s the package.json test script used before the repoint', () => {
|
||||
expect(FREE_TEST_TIMEOUT_MS).toBe(30_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('test-free-shards: strict shard execution', () => {
|
||||
// Fake command seam, same pattern as test/paid-shards.test.ts: each "file"
|
||||
// label selects a child command. Unlike the paid runner, runFreeShard
|
||||
// enforces the terminal-summary file count on injected commands too, so
|
||||
// fake PASSING commands must print a synthetic bun summary line.
|
||||
const SUMMARY_1 = 'Ran 3 tests across 1 files. [12.00ms]';
|
||||
const BUSY_LOOP = 'const end = Date.now() + 600000; while (Date.now() < end) {}';
|
||||
const FAIL_LINE = '(fa' + 'il) planted failure [0.10ms]'; // split so this source file never contains a raw bun fail line
|
||||
|
||||
const commandFor = (files: string[]) => {
|
||||
const mode = files[0];
|
||||
if (mode === 'spin') return { command: process.execPath, args: ['-e', BUSY_LOOP] };
|
||||
if (mode === 'no-summary') return { command: process.execPath, args: ['-e', 'console.log("ok")'] };
|
||||
if (mode === 'fail-exit') {
|
||||
return { command: process.execPath, args: ['-e', `console.log(${JSON.stringify(SUMMARY_1)}); process.exit(3)`] };
|
||||
}
|
||||
if (mode === 'fail-line-exit-zero') {
|
||||
return { command: process.execPath, args: ['-e', `console.log(${JSON.stringify(FAIL_LINE)}); console.log(${JSON.stringify(SUMMARY_1)})`] };
|
||||
}
|
||||
if (mode === 'wrong-file-count') {
|
||||
return { command: process.execPath, args: ['-e', 'console.log("Ran 3 tests across 4 files. [12.00ms]")'] };
|
||||
}
|
||||
return { command: process.execPath, args: ['-e', `console.log(${JSON.stringify(SUMMARY_1)})`] };
|
||||
};
|
||||
|
||||
test('exit 0 WITHOUT bun\'s terminal summary is a FAILURE (anti-truncation backstop)', async () => {
|
||||
const outcome = await runFreeShard(['no-summary'], 1, 1, { commandFor, quiet: true, log: () => {} });
|
||||
expect(outcome.status).toBe('failed');
|
||||
expect(outcome.exitCode).toBe(0);
|
||||
});
|
||||
|
||||
test('exit 0 WITH the terminal summary passes, and the per-shard epilogue line is printed', async () => {
|
||||
const lines: string[] = [];
|
||||
const outcome = await runFreeShard(['pass'], 1, 1, { commandFor, quiet: true, log: (l) => lines.push(l) });
|
||||
expect(outcome.status).toBe('passed');
|
||||
expect(lines.some((l) => /^\[test:free\] shard 1\/1: 1 files, \d+s, pass$/.test(l))).toBe(true);
|
||||
});
|
||||
|
||||
test('a non-zero exit stays a failure even when the summary is present', async () => {
|
||||
const outcome = await runFreeShard(['fail-exit'], 1, 1, { commandFor, quiet: true, log: () => {} });
|
||||
expect(outcome.status).toBe('failed');
|
||||
expect(outcome.exitCode).toBe(3);
|
||||
});
|
||||
|
||||
test('a printed (fail) result line is a failure even on exit 0 (bun exit-code bug class)', async () => {
|
||||
const outcome = await runFreeShard(['fail-line-exit-zero'], 1, 1, { commandFor, quiet: true, log: () => {} });
|
||||
expect(outcome.status).toBe('failed');
|
||||
expect(outcome.exitCode).toBe(0);
|
||||
});
|
||||
|
||||
test('a summary reporting the wrong file count is a failure (partial execution)', async () => {
|
||||
const outcome = await runFreeShard(['wrong-file-count'], 1, 1, { commandFor, quiet: true, log: () => {} });
|
||||
expect(outcome.status).toBe('failed');
|
||||
});
|
||||
|
||||
test('a spinning shard is killed at the wall-clock deadline and reported timed-out, distinct from failed', async () => {
|
||||
const lines: string[] = [];
|
||||
const outcome = await runFreeShard(['spin'], 1, 1, {
|
||||
commandFor, quiet: true, wallTimeoutMs: 1_200, log: (l) => lines.push(l),
|
||||
});
|
||||
expect(outcome.status).toBe('timed-out');
|
||||
expect(outcome.status).not.toBe('failed');
|
||||
// Killed at the deadline, not left to burn the full 600s busy loop.
|
||||
expect(outcome.elapsedMs).toBeLessThan(30_000);
|
||||
expect(outcome.groupPid).toBeGreaterThan(0);
|
||||
if (process.platform !== 'win32') {
|
||||
expect(() => process.kill(outcome.groupPid as number, 0)).toThrow();
|
||||
}
|
||||
expect(lines.some((l) => /^\[test:free\] shard 1\/1: 1 files, \d+s, timed-out$/.test(l))).toBe(true);
|
||||
}, 30_000);
|
||||
|
||||
test('an empty shard is a fast no-op success and never spawns (stable CI-matrix indices)', async () => {
|
||||
const lines: string[] = [];
|
||||
const outcome = await runFreeShard([], 7, 20, {
|
||||
commandFor: () => { throw new Error('an empty shard must not spawn a child'); },
|
||||
log: (l) => lines.push(l),
|
||||
});
|
||||
expect(outcome.status).toBe('passed');
|
||||
expect(lines.some((l) => /^\[test:free\] shard 7\/20: 0 files, 0s, pass$/.test(l))).toBe(true);
|
||||
});
|
||||
|
||||
test('the log-file path is announced once at start and the PASS epilogue repeats it', async () => {
|
||||
const lines: string[] = [];
|
||||
const outcome = await runFreeShard(['pass'], 1, 1, { commandFor, quiet: true, log: (l) => lines.push(l) });
|
||||
expect(outcome.status).toBe('passed');
|
||||
const announced = lines.filter((l) => /^\[test:free\] full log: .+gstack-free-test-.+\.log$/.test(l));
|
||||
expect(announced.length).toBe(1);
|
||||
// PASS epilogue carries the counts from the terminal summary + the log path.
|
||||
expect(lines.some((l) => /^\[test:free\] PASS — 3 tests, 1 files, \d+s\. Full log: .+\.log$/.test(l))).toBe(true);
|
||||
});
|
||||
|
||||
test('spawned shard gets throwaway TMPDIR but NEVER an injected GSTACK_HOME', async () => {
|
||||
// GSTACK_HOME injection was tried and reverted: one shared scratch home
|
||||
// per invocation made 6,900 tests share MUTABLE state — config tests
|
||||
// wrote keys that relink/update-check tests then read (12 measured
|
||||
// cross-contamination failures). This pin keeps the regression out.
|
||||
const captureDir = fs.mkdtempSync(path.join(os.tmpdir(), 'free-shard-env-'));
|
||||
const dump = path.join(captureDir, 'env.json');
|
||||
try {
|
||||
const script =
|
||||
`const fs = require("fs");`
|
||||
+ `fs.writeFileSync(${JSON.stringify(dump)}, JSON.stringify({`
|
||||
+ ` home: process.env.GSTACK_HOME ?? null, tmp: process.env.TMPDIR,`
|
||||
+ ` tmpExists: fs.existsSync(process.env.TMPDIR || "") }));`
|
||||
+ `console.log(${JSON.stringify(SUMMARY_1)});`;
|
||||
const outcome = await runFreeShard(['env-dump'], 1, 1, {
|
||||
commandFor: () => ({ command: process.execPath, args: ['-e', script] }),
|
||||
quiet: true,
|
||||
log: () => {},
|
||||
});
|
||||
expect(outcome.status).toBe('passed');
|
||||
const seen = JSON.parse(fs.readFileSync(dump, 'utf8'));
|
||||
// GSTACK_HOME passes through untouched (whatever the parent had, incl. unset).
|
||||
expect(seen.home).toBe(process.env.GSTACK_HOME ?? null);
|
||||
// TMPDIR is a per-shard throwaway, cleaned up once the shard finishes.
|
||||
expect(seen.tmp).toContain('gstack-free-shard-');
|
||||
expect(seen.tmpExists).toBe(true);
|
||||
expect(seen.tmp).not.toBe(process.env.TMPDIR ?? '');
|
||||
expect(fs.existsSync(seen.tmp)).toBe(false);
|
||||
} finally {
|
||||
fs.rmSync(captureDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('test-free-shards: output contract (log capture, quiet console, failure epilogue)', () => {
|
||||
// Convention from the block above: never write a raw bun fail line into this
|
||||
// source file — build it at runtime so a printed source excerpt can't trip
|
||||
// the strict classifier.
|
||||
const FAIL_WORD = '(fa' + 'il)';
|
||||
const failLine = (name: string) => `${FAIL_WORD} ${name} [0.10ms]`;
|
||||
const SUMMARY_1 = 'Ran 3 tests across 1 files. [12.00ms]';
|
||||
|
||||
/** Fake child that prints the given lines (stdout, then stderr) and exits. */
|
||||
const commandPrinting = (stdoutLines: string[], stderrLines: string[] = [], exitCode = 0) => () => ({
|
||||
command: process.execPath,
|
||||
args: ['-e',
|
||||
stdoutLines.map((l) => `console.log(${JSON.stringify(l)});`).join('')
|
||||
+ stderrLines.map((l) => `console.error(${JSON.stringify(l)});`).join('')
|
||||
+ (exitCode !== 0 ? `process.exit(${exitCode});` : ''),
|
||||
],
|
||||
});
|
||||
|
||||
test('failure epilogue names the failing test, attributed to its file-chunk header', async () => {
|
||||
const lines: string[] = [];
|
||||
const commandFor = commandPrinting(['test/planted.test.ts:', failLine('planted failure'), SUMMARY_1]);
|
||||
const outcome = await runFreeShard(['planted'], 1, 1, { commandFor, quiet: true, log: (l) => lines.push(l) });
|
||||
expect(outcome.status).toBe('failed');
|
||||
expect(lines.some((l) =>
|
||||
/^\[test:free\] FAIL — 1 failing test\(s\) in 1 file\(s\), 0 crashed worker\(s\)\. Full log: .+\.log$/.test(l),
|
||||
)).toBe(true);
|
||||
expect(lines).toContain(' ✗ test/planted.test.ts — planted failure');
|
||||
});
|
||||
|
||||
test('crash markers surface in the epilogue as crashed+retried workers', async () => {
|
||||
const lines: string[] = [];
|
||||
const commandFor = commandPrinting([
|
||||
'test/crashy.test.ts:',
|
||||
'⟳ crashed running test/crashy.test.ts, retrying',
|
||||
'test/crashy.test.ts:',
|
||||
'✗ test/crashy.test.ts (crashed: exited)',
|
||||
'Ran 0 tests across 1 files. [12.00ms]',
|
||||
], [], 1);
|
||||
const outcome = await runFreeShard(['crashy'], 1, 1, { commandFor, quiet: true, log: (l) => lines.push(l) });
|
||||
expect(outcome.status).toBe('failed');
|
||||
expect(lines.some((l) =>
|
||||
/^\[test:free\] FAIL — 0 failing test\(s\) in 0 file\(s\), 1 crashed worker\(s\)\. Full log: /.test(l),
|
||||
)).toBe(true);
|
||||
expect(lines).toContain(' ⚠ crashed+retried: test/crashy.test.ts');
|
||||
});
|
||||
|
||||
test('default console is quiet: noise stays in the log; fail/error/summary lines pass through', async () => {
|
||||
const consoleOut: string[] = [];
|
||||
const commandFor = commandPrinting([
|
||||
'PASSING-NOISE gitleaks ascii art',
|
||||
'test/noisy.test.ts:',
|
||||
failLine('quiet mode failure'),
|
||||
'error: expect(received).toBe(expected)',
|
||||
'Ran 1 tests across 1 files. [1.00ms]',
|
||||
], ['telemetry stderr spam']);
|
||||
const outcome = await runFreeShard(['noisy'], 1, 1, {
|
||||
commandFor, consoleWrite: (t) => consoleOut.push(t), log: () => {},
|
||||
});
|
||||
expect(outcome.status).toBe('failed');
|
||||
const joined = consoleOut.join('');
|
||||
expect(joined).toContain(failLine('quiet mode failure'));
|
||||
expect(joined).toContain('error: expect(received).toBe(expected)');
|
||||
expect(joined).toContain('Ran 1 tests across 1 files.');
|
||||
expect(joined).not.toContain('PASSING-NOISE');
|
||||
expect(joined).not.toContain('telemetry stderr spam');
|
||||
expect(joined).not.toContain('test/noisy.test.ts:'); // headers feed the epilogue, not the console
|
||||
});
|
||||
|
||||
test('--verbose restores the full firehose to the console', async () => {
|
||||
const consoleOut: string[] = [];
|
||||
const commandFor = commandPrinting(
|
||||
['PASSING-NOISE gitleaks ascii art', SUMMARY_1],
|
||||
['telemetry stderr spam'],
|
||||
);
|
||||
const outcome = await runFreeShard(['pass'], 1, 1, {
|
||||
commandFor, verbose: true, consoleWrite: (t) => consoleOut.push(t), log: () => {},
|
||||
});
|
||||
expect(outcome.status).toBe('passed');
|
||||
const joined = consoleOut.join('');
|
||||
expect(joined).toContain('PASSING-NOISE gitleaks ascii art');
|
||||
expect(joined).toContain('telemetry stderr spam');
|
||||
});
|
||||
|
||||
test('quiet suppresses the console entirely, even with an injected sink', async () => {
|
||||
const consoleOut: string[] = [];
|
||||
const commandFor = commandPrinting(['PASSING-NOISE', failLine('hidden'), SUMMARY_1]);
|
||||
await runFreeShard(['pass'], 1, 1, {
|
||||
commandFor, quiet: true, consoleWrite: (t) => consoleOut.push(t), log: () => {},
|
||||
});
|
||||
expect(consoleOut).toEqual([]);
|
||||
});
|
||||
|
||||
test('the full child stream lands in the per-run log file, including console-filtered noise', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'free-log-'));
|
||||
const logFilePath = path.join(dir, 'run.log');
|
||||
try {
|
||||
const lines: string[] = [];
|
||||
const commandFor = commandPrinting(['stdout NOISE-A', SUMMARY_1], ['stderr NOISE-B']);
|
||||
const outcome = await runFreeShard(['pass'], 1, 1, { commandFor, quiet: true, logFilePath, log: (l) => lines.push(l) });
|
||||
expect(outcome.status).toBe('passed');
|
||||
expect(lines).toContain(`[test:free] full log: ${logFilePath}`);
|
||||
const logged = fs.readFileSync(logFilePath, 'utf8');
|
||||
expect(logged).toContain('stdout NOISE-A');
|
||||
expect(logged).toContain('stderr NOISE-B');
|
||||
expect(logged).toContain('Ran 3 tests across 1 files.');
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('colored fail lines are attributed after ANSI stripping (a prior grep missed them)', async () => {
|
||||
const lines: string[] = [];
|
||||
const colored = `\u001B[31m${failLine('colored failure')}\u001B[0m`;
|
||||
const commandFor = commandPrinting(['test/colored.test.ts:', colored, 'Ran 1 tests across 1 files. [1.00ms]']);
|
||||
const outcome = await runFreeShard(['colored'], 1, 1, { commandFor, quiet: true, log: (l) => lines.push(l) });
|
||||
expect(outcome.status).toBe('failed');
|
||||
expect(lines).toContain(' ✗ test/colored.test.ts — colored failure');
|
||||
});
|
||||
|
||||
test('wall-timeout epilogue lists wedge suspects: header seen, no results, no summary', async () => {
|
||||
const lines: string[] = [];
|
||||
const commandFor = () => ({
|
||||
command: process.execPath,
|
||||
args: ['-e', 'console.log("test/wedged.test.ts:");console.log("wedged noise");setTimeout(() => {}, 600000);'],
|
||||
});
|
||||
const outcome = await runFreeShard(['wedged'], 1, 1, {
|
||||
commandFor, quiet: true, wallTimeoutMs: 1_500, log: (l) => lines.push(l),
|
||||
});
|
||||
expect(outcome.status).toBe('timed-out');
|
||||
expect(lines).toContain(' ⏱ in flight at kill: test/wedged.test.ts');
|
||||
// The epilogue headline shape stays stable across statuses.
|
||||
expect(lines.some((l) => l.startsWith('[test:free] FAIL — '))).toBe(true);
|
||||
}, 30_000);
|
||||
|
||||
test('timeout with no observable header falls back to the buffered-parallel explanation', () => {
|
||||
const reporter = new FreeRunReporter(['test/a.test.ts', 'test/b.test.ts']);
|
||||
reporter.end();
|
||||
const lines = buildRunEpilogue('timed-out', reporter.report(), 5_000, '/tmp/x.log');
|
||||
expect(lines.some((l) => l.includes('in flight at kill: unknown'))).toBe(true);
|
||||
expect(lines.some((l) => l.includes('2 planned file(s) produced no output'))).toBe(true);
|
||||
});
|
||||
|
||||
test('duplicate fail lines dedupe; pre-header failures are labeled unattributed', () => {
|
||||
const reporter = new FreeRunReporter(['test/a.test.ts']);
|
||||
reporter.write(`${failLine('early unattributed')}\n`, 'stderr');
|
||||
reporter.write('test/a.test.ts:\n', 'stderr');
|
||||
reporter.write(`${failLine('dup')}\n${failLine('dup')}\n`, 'stderr');
|
||||
reporter.end();
|
||||
const report = reporter.report();
|
||||
expect(report.failures).toEqual([
|
||||
{ file: null, testName: 'early unattributed' },
|
||||
{ file: 'test/a.test.ts', testName: 'dup' },
|
||||
]);
|
||||
const lines = buildRunEpilogue('failed', report, 1_000, '/tmp/x.log');
|
||||
expect(lines).toContain(' ✗ (unattributed) — early unattributed');
|
||||
expect(lines).toContain(' ✗ test/a.test.ts — dup');
|
||||
expect(lines.some((l) => l.includes('2 failing test(s) in 2 file(s)'))).toBe(true);
|
||||
});
|
||||
|
||||
test("a later file's header ends the previous chunk — completed noisy files are not wedge suspects", () => {
|
||||
const reporter = new FreeRunReporter(['test/done.test.ts', 'test/hung.test.ts']);
|
||||
reporter.write('test/done.test.ts:\n', 'stderr');
|
||||
reporter.write('noise from the completed file\n', 'stderr');
|
||||
reporter.write('test/hung.test.ts:\n', 'stderr');
|
||||
reporter.write('noise before the hang\n', 'stderr');
|
||||
reporter.end();
|
||||
// No terminal summary: only the still-open chunk is in flight.
|
||||
expect(reporter.report().inFlight).toEqual(['test/hung.test.ts']);
|
||||
});
|
||||
|
||||
test('../-prefixed printed paths canonicalize to planned relative paths (symlinked cwd)', () => {
|
||||
const reporter = new FreeRunReporter(['browse/test/x.test.ts']);
|
||||
reporter.write('../../../work/repo/browse/test/x.test.ts:\n', 'stderr');
|
||||
reporter.write(`${failLine('boom')}\n`, 'stderr');
|
||||
reporter.end();
|
||||
expect(reporter.report().failures[0]).toEqual({ file: 'browse/test/x.test.ts', testName: 'boom' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('test-free-shards: GitHub Actions log-group attribution', () => {
|
||||
const failLine = (name: string) => `(fail) ${name} [1.00ms]`;
|
||||
// On GHA (GITHUB_ACTIONS=1) bun wraps each file's section in ::group::.
|
||||
// Unstripped, the real header fails FILE_HEADER_RE, failures attribute to
|
||||
// the PREVIOUS file, and the terminal recap's re-printed (fail) lines land
|
||||
// under a phantom second file — the first Linux run reported 5 real
|
||||
// failures as 10 across 2 files.
|
||||
test('::group::-wrapped headers attribute failures to the right file, once', () => {
|
||||
const reporter = new FreeRunReporter(['test/a.test.ts', 'test/b.test.ts']);
|
||||
reporter.write('::group::test/a.test.ts:\n', 'stderr');
|
||||
reporter.write('::endgroup::\n', 'stderr');
|
||||
reporter.write('::group::test/b.test.ts:\n', 'stderr');
|
||||
reporter.write(`${failLine('planted')}\n`, 'stderr');
|
||||
reporter.write('::endgroup::\n', 'stderr');
|
||||
// Terminal recap re-prints the failing file header + result line.
|
||||
reporter.write('1 tests failed:\n', 'stderr');
|
||||
reporter.write('::group::test/b.test.ts:\n', 'stderr');
|
||||
reporter.write(`${failLine('planted')}\n`, 'stderr');
|
||||
reporter.end();
|
||||
expect(reporter.report().failures).toEqual([{ file: 'test/b.test.ts', testName: 'planted' }]);
|
||||
});
|
||||
|
||||
test('headerless recap re-prints do not invent a phantom failing file', () => {
|
||||
// Round-3 CI shape: bun's recap prints "N tests failed:" then the (fail)
|
||||
// lines with NO file headers — the stale currentFile (an innocent file)
|
||||
// was charged with the previous file's failures.
|
||||
const reporter = new FreeRunReporter(['test/a.test.ts', 'test/b.test.ts']);
|
||||
reporter.write('::group::test/a.test.ts:\n', 'stderr');
|
||||
reporter.write(`${failLine('planted')}\n`, 'stderr');
|
||||
reporter.write('::endgroup::\n', 'stderr');
|
||||
reporter.write('::group::test/b.test.ts:\n', 'stderr');
|
||||
reporter.write('(pass-ish output, no failures here)\n', 'stderr');
|
||||
reporter.write('2 tests failed:\n', 'stderr');
|
||||
reporter.write(`${failLine('planted')}\n`, 'stderr');
|
||||
reporter.write(`${failLine('planted')}\n`, 'stderr');
|
||||
reporter.end();
|
||||
expect(reporter.report().failures).toEqual([{ file: 'test/a.test.ts', testName: 'planted' }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('test-free-shards: curated-list census pins', () => {
|
||||
// A renamed test file must FAIL here, not silently drop its serialization
|
||||
// (a phantom TREE_MUTATING key means the reader races regenerating shards
|
||||
// again) or its serial-child quarantine (WORKER_HOSTILE).
|
||||
test('every TREE_MUTATING and WORKER_HOSTILE key names a real free test file', () => {
|
||||
const census = new Set(collectFreeTestFiles(ROOT));
|
||||
const stale = [...Object.keys(TREE_MUTATING), ...Object.keys(WORKER_HOSTILE)]
|
||||
.filter((key) => !census.has(key));
|
||||
expect(stale).toEqual([]);
|
||||
});
|
||||
|
||||
test('every KNOWN_WINDOWS_INCOMPATIBLE entry names a real free test file', () => {
|
||||
const census = new Set(collectFreeTestFiles(ROOT));
|
||||
const stale = KNOWN_WINDOWS_INCOMPATIBLE.map((e) => e.file).filter((f) => !census.has(f));
|
||||
expect(stale).toEqual([]);
|
||||
});
|
||||
|
||||
test('every TEST_ROOTS entry exists on disk and contributes at least one test file', () => {
|
||||
const files = collectFreeTestFiles(ROOT);
|
||||
for (const root of TEST_ROOTS) {
|
||||
expect(fs.existsSync(path.join(ROOT, root))).toBe(true);
|
||||
expect(files.some((f) => f.startsWith(`${root}/`))).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('test-free-shards: wall-timeout scaling', () => {
|
||||
test('typical local shard keeps the 6-minute floor', () => {
|
||||
expect(wallTimeoutForShard(70)).toBe(DEFAULT_WALL_TIMEOUT_MS);
|
||||
});
|
||||
|
||||
test('oversized shards (jobs=1 machines, Windows lane) scale linearly past the floor', () => {
|
||||
expect(wallTimeoutForShard(130)).toBe(130 * PER_FILE_WALL_MS);
|
||||
expect(wallTimeoutForShard(420)).toBe(420 * PER_FILE_WALL_MS);
|
||||
});
|
||||
|
||||
test('an explicit base above the scaled value wins', () => {
|
||||
expect(wallTimeoutForShard(10, 10 * 60_000)).toBe(10 * 60_000);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
/**
|
||||
* Pins the touchfiles three-file split (facade + data + logic).
|
||||
* Free (no API calls), runs with `bun test`.
|
||||
*
|
||||
* (a) touchfiles-data.ts stays LITERALS ONLY — no imports/requires, no call
|
||||
* expressions, no spreads, no template literals. Map-diff selection
|
||||
* evaluates old git versions of that file standalone; any logic breaks it.
|
||||
* (b) the ./helpers/touchfiles facade re-exports EVERY export of both halves
|
||||
* by identity (===), so existing import sites see the same objects.
|
||||
* (c) the data file's exports are importable and non-empty.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { readFileSync } from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
import * as data from './helpers/touchfiles-data';
|
||||
import * as logic from './helpers/test-selection';
|
||||
import * as facade from './helpers/touchfiles';
|
||||
|
||||
const DATA_PATH = path.join(import.meta.dir, 'helpers', 'touchfiles-data.ts');
|
||||
|
||||
// Single-pass scanner: returns the source with comments AND string literals
|
||||
// removed (only code-position characters survive), plus whether any backtick
|
||||
// appeared in code position.
|
||||
//
|
||||
// Why a state machine instead of regexes: the data strings contain glob
|
||||
// patterns like 'browse/src/' + '**' (a block-comment OPENER to a naive
|
||||
// regex) and '*' + '/SKILL.md.tmpl' (a block-comment CLOSER), so regex
|
||||
// comment-stripping would treat string content as comment delimiters.
|
||||
// Conversely, comments contain apostrophes ("the model's interpretation"),
|
||||
// so regex string-stripping applied first would eat code. Tracking state
|
||||
// character-by-character handles both without false positives.
|
||||
function stripCommentsAndStrings(src: string): { code: string; sawBacktick: boolean } {
|
||||
let code = '';
|
||||
let sawBacktick = false;
|
||||
let state: 'code' | 'line' | 'block' | 'single' | 'double' = 'code';
|
||||
let i = 0;
|
||||
while (i < src.length) {
|
||||
const c = src[i];
|
||||
const n = src[i + 1];
|
||||
if (state === 'code') {
|
||||
if (c === '/' && n === '/') { state = 'line'; i += 2; continue; }
|
||||
if (c === '/' && n === '*') { state = 'block'; i += 2; continue; }
|
||||
if (c === "'") { state = 'single'; i += 1; continue; }
|
||||
if (c === '"') { state = 'double'; i += 1; continue; }
|
||||
if (c === '`') { sawBacktick = true; i += 1; continue; }
|
||||
code += c;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (state === 'line') {
|
||||
if (c === '\n') { state = 'code'; code += '\n'; }
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if (state === 'block') {
|
||||
if (c === '*' && n === '/') { state = 'code'; i += 2; } else { i += 1; }
|
||||
continue;
|
||||
}
|
||||
// single- or double-quoted string
|
||||
if (c === '\\') { i += 2; continue; }
|
||||
if ((state === 'single' && c === "'") || (state === 'double' && c === '"')) {
|
||||
state = 'code';
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
return { code, sawBacktick };
|
||||
}
|
||||
|
||||
describe('touchfiles-data.ts literal-only tripwire', () => {
|
||||
const src = readFileSync(DATA_PATH, 'utf-8');
|
||||
const { code, sawBacktick } = stripCommentsAndStrings(src);
|
||||
const explain =
|
||||
'touchfiles-data.ts must stay LITERALS ONLY (map-diff selection evaluates ' +
|
||||
'old git versions of it standalone). Move any logic to test-selection.ts.';
|
||||
|
||||
test('scanner sanity: keeps code, strips comments and strings', () => {
|
||||
const sample =
|
||||
"export const X = { 'a(b)': ['c/**'] }; // call() here\n" +
|
||||
'/* import */ const Y = 1; // `tpl`\n';
|
||||
const out = stripCommentsAndStrings(sample);
|
||||
expect(out.code).toContain('export const X');
|
||||
expect(out.code).toContain('const Y = 1');
|
||||
expect(out.code).not.toContain('call(');
|
||||
expect(out.code).not.toContain('import');
|
||||
expect(out.code).not.toContain('a(b)'); // string content stripped
|
||||
expect(out.sawBacktick).toBe(false); // backticks inside comments don't count
|
||||
});
|
||||
|
||||
test('no import or require statements', () => {
|
||||
expect(code, explain).not.toMatch(/\bimport\b/);
|
||||
expect(code, explain).not.toMatch(/\brequire\b/);
|
||||
});
|
||||
|
||||
test('no spread operator', () => {
|
||||
expect(code, explain).not.toContain('...');
|
||||
});
|
||||
|
||||
test('no call expressions', () => {
|
||||
expect(code, explain).not.toMatch(/[A-Za-z_$][A-Za-z0-9_$]*\s*\(/);
|
||||
});
|
||||
|
||||
test('no template literals or interpolation', () => {
|
||||
expect(sawBacktick, explain).toBe(false);
|
||||
expect(code, explain).not.toContain('${');
|
||||
});
|
||||
|
||||
test('no duplicate keys within any map block', () => {
|
||||
// JS object evaluation silently keeps the LAST duplicate — the earlier
|
||||
// dep list becomes dead weight an editor can update to no effect, and
|
||||
// no runtime assertion can see the collapsed key. Scan the source.
|
||||
const blocks = src.split(/export const /).slice(1);
|
||||
const dupes: string[] = [];
|
||||
for (const block of blocks) {
|
||||
const name = block.slice(0, block.indexOf(' '));
|
||||
const seen = new Set<string>();
|
||||
for (const match of block.matchAll(/^\s{2}'([^']+)':/gm)) {
|
||||
if (seen.has(match[1])) dupes.push(`${name}: '${match[1]}'`);
|
||||
seen.add(match[1]);
|
||||
}
|
||||
}
|
||||
expect(dupes, 'duplicate keys collapse silently — the earlier entry is dead').toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('facade export parity', () => {
|
||||
test('every touchfiles-data export is re-exported by identity', () => {
|
||||
const dataExports = Object.keys(data);
|
||||
expect(dataExports.length).toBeGreaterThan(0);
|
||||
for (const name of dataExports) {
|
||||
expect(
|
||||
(facade as Record<string, unknown>)[name],
|
||||
`facade must re-export '${name}' from touchfiles-data by identity`,
|
||||
).toBe((data as Record<string, unknown>)[name] as never);
|
||||
}
|
||||
});
|
||||
|
||||
test('every test-selection export is re-exported by identity', () => {
|
||||
const logicExports = Object.keys(logic);
|
||||
expect(logicExports.length).toBeGreaterThan(0);
|
||||
for (const name of logicExports) {
|
||||
expect(
|
||||
(facade as Record<string, unknown>)[name],
|
||||
`facade must re-export '${name}' from test-selection by identity`,
|
||||
).toBe((logic as Record<string, unknown>)[name] as never);
|
||||
}
|
||||
});
|
||||
|
||||
test('facade exports exactly the union of both halves', () => {
|
||||
const union = new Set([...Object.keys(data), ...Object.keys(logic)]);
|
||||
expect(new Set(Object.keys(facade))).toEqual(union);
|
||||
});
|
||||
|
||||
test('no export name collisions between data and logic', () => {
|
||||
const overlap = Object.keys(data).filter((k) => k in logic);
|
||||
expect(overlap).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('touchfiles-data exports are importable and non-empty', () => {
|
||||
test('E2E_TOUCHFILES has entries with non-empty pattern lists', () => {
|
||||
const keys = Object.keys(data.E2E_TOUCHFILES);
|
||||
expect(keys.length).toBeGreaterThan(0);
|
||||
for (const key of keys) {
|
||||
expect(data.E2E_TOUCHFILES[key].length, `E2E_TOUCHFILES['${key}'] is empty`).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
test('E2E_TIERS has entries with valid tier values', () => {
|
||||
const entries = Object.entries(data.E2E_TIERS);
|
||||
expect(entries.length).toBeGreaterThan(0);
|
||||
for (const [key, tier] of entries) {
|
||||
expect(['gate', 'periodic'], `E2E_TIERS['${key}'] has invalid tier`).toContain(tier);
|
||||
}
|
||||
});
|
||||
|
||||
test('LLM_JUDGE_TOUCHFILES has entries with non-empty pattern lists', () => {
|
||||
const keys = Object.keys(data.LLM_JUDGE_TOUCHFILES);
|
||||
expect(keys.length).toBeGreaterThan(0);
|
||||
for (const key of keys) {
|
||||
expect(data.LLM_JUDGE_TOUCHFILES[key].length, `LLM_JUDGE_TOUCHFILES['${key}'] is empty`).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
test('GLOBAL_TOUCHFILES is non-empty and covers the selection logic', () => {
|
||||
expect(data.GLOBAL_TOUCHFILES.length).toBeGreaterThan(0);
|
||||
// The logic file must stay a global touchfile: a bug in selectTests /
|
||||
// matchGlob mis-selects every test, so any change to it forces a full run.
|
||||
expect(data.GLOBAL_TOUCHFILES).toContain('test/helpers/test-selection.ts');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,360 @@
|
||||
/**
|
||||
* Map-diff selection for touchfiles-data.ts changes.
|
||||
* Free (no API calls), runs with `bun test`.
|
||||
*
|
||||
* Three layers, matching the injectable-core + thin-shell shape:
|
||||
* 1. diffTouchfileMapsCore — pure diff logic on injected old/new maps.
|
||||
* 2. selectTests wiring — injected MapDiffOutcome, no git.
|
||||
* 3. diffTouchfileMaps shell — real git + bun-child evaluation against a
|
||||
* throwaway temp repo (happy path + every fail-closed cause), plus one
|
||||
* end-to-end call against this actual repo's HEAD.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
|
||||
import {
|
||||
diffTouchfileMaps,
|
||||
diffTouchfileMapsCore,
|
||||
selectTests,
|
||||
TOUCHFILES_DATA_PATH,
|
||||
E2E_TOUCHFILES,
|
||||
GLOBAL_TOUCHFILES,
|
||||
} from './helpers/touchfiles';
|
||||
import type { TouchfileMaps, MapDiffOutcome } from './helpers/touchfiles';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
function maps(overrides: Partial<TouchfileMaps> = {}): TouchfileMaps {
|
||||
return {
|
||||
E2E_TOUCHFILES: {
|
||||
'alpha': ['a/**'],
|
||||
'beta': ['b/**', 'shared/util.ts'],
|
||||
},
|
||||
E2E_TIERS: {
|
||||
'alpha': 'gate',
|
||||
'beta': 'periodic',
|
||||
},
|
||||
LLM_JUDGE_TOUCHFILES: {
|
||||
'judge one': ['j/SKILL.md'],
|
||||
},
|
||||
GLOBAL_TOUCHFILES: ['test/helpers/session-runner.ts'],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// --- Layer 1: pure core ---
|
||||
|
||||
describe('diffTouchfileMapsCore', () => {
|
||||
test('identical maps → nothing changed', () => {
|
||||
const result = diffTouchfileMapsCore(maps(), maps());
|
||||
expect(result.changedTests).toEqual([]);
|
||||
expect(result.removedTests).toEqual([]);
|
||||
expect(result.globalTouchfilesChanged).toBe(false);
|
||||
});
|
||||
|
||||
test('entry added → changed', () => {
|
||||
const newMaps = maps({
|
||||
E2E_TOUCHFILES: { 'alpha': ['a/**'], 'beta': ['b/**', 'shared/util.ts'], 'gamma': ['g/**'] },
|
||||
E2E_TIERS: { 'alpha': 'gate', 'beta': 'periodic', 'gamma': 'gate' },
|
||||
});
|
||||
const result = diffTouchfileMapsCore(maps(), newMaps);
|
||||
expect(result.changedTests).toEqual(['gamma']);
|
||||
expect(result.removedTests).toEqual([]);
|
||||
});
|
||||
|
||||
test('dep glob edited → changed', () => {
|
||||
const newMaps = maps({
|
||||
E2E_TOUCHFILES: { 'alpha': ['a/**', 'extra/dep.ts'], 'beta': ['b/**', 'shared/util.ts'] },
|
||||
});
|
||||
const result = diffTouchfileMapsCore(maps(), newMaps);
|
||||
expect(result.changedTests).toEqual(['alpha']);
|
||||
});
|
||||
|
||||
test('tier flipped → changed', () => {
|
||||
const newMaps = maps({
|
||||
E2E_TIERS: { 'alpha': 'gate', 'beta': 'gate' },
|
||||
});
|
||||
const result = diffTouchfileMapsCore(maps(), newMaps);
|
||||
expect(result.changedTests).toEqual(['beta']);
|
||||
});
|
||||
|
||||
test('unrelated entries untouched → not selected', () => {
|
||||
const newMaps = maps({
|
||||
E2E_TOUCHFILES: { 'alpha': ['a/**', 'x.ts'], 'beta': ['b/**', 'shared/util.ts'] },
|
||||
});
|
||||
const result = diffTouchfileMapsCore(maps(), newMaps);
|
||||
expect(result.changedTests).not.toContain('beta');
|
||||
expect(result.changedTests).not.toContain('judge one');
|
||||
});
|
||||
|
||||
test('entry removed from every map → removedTests, not changed', () => {
|
||||
const newMaps = maps({
|
||||
E2E_TOUCHFILES: { 'alpha': ['a/**'] },
|
||||
E2E_TIERS: { 'alpha': 'gate' },
|
||||
});
|
||||
const result = diffTouchfileMapsCore(maps(), newMaps);
|
||||
expect(result.removedTests).toEqual(['beta']);
|
||||
expect(result.changedTests).not.toContain('beta');
|
||||
});
|
||||
|
||||
test('tier entry removed but touchfile entry kept → changed (conservative)', () => {
|
||||
const newMaps = maps({
|
||||
E2E_TIERS: { 'alpha': 'gate' }, // 'beta' tier dropped, E2E_TOUCHFILES.beta kept
|
||||
});
|
||||
const result = diffTouchfileMapsCore(maps(), newMaps);
|
||||
expect(result.changedTests).toContain('beta');
|
||||
expect(result.removedTests).toEqual([]);
|
||||
});
|
||||
|
||||
test('LLM-judge entries participate in the diff', () => {
|
||||
const newMaps = maps({
|
||||
LLM_JUDGE_TOUCHFILES: { 'judge one': ['j/SKILL.md', 'j/SKILL.md.tmpl'] },
|
||||
});
|
||||
const result = diffTouchfileMapsCore(maps(), newMaps);
|
||||
expect(result.changedTests).toEqual(['judge one']);
|
||||
});
|
||||
|
||||
test('GLOBAL_TOUCHFILES entry added → flagged', () => {
|
||||
const newMaps = maps({
|
||||
GLOBAL_TOUCHFILES: ['test/helpers/session-runner.ts', 'test/helpers/new-global.ts'],
|
||||
});
|
||||
const result = diffTouchfileMapsCore(maps(), newMaps);
|
||||
expect(result.globalTouchfilesChanged).toBe(true);
|
||||
});
|
||||
|
||||
test('GLOBAL_TOUCHFILES compared as a set — reorder is not a change', () => {
|
||||
const oldMaps = maps({ GLOBAL_TOUCHFILES: ['x.ts', 'y.ts'] });
|
||||
const newMaps = maps({ GLOBAL_TOUCHFILES: ['y.ts', 'x.ts'] });
|
||||
const result = diffTouchfileMapsCore(oldMaps, newMaps);
|
||||
expect(result.globalTouchfilesChanged).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// --- Layer 2: selectTests wiring (injected outcome, no git) ---
|
||||
|
||||
describe('selectTests map-diff wiring', () => {
|
||||
const okOutcome = (changedTests: string[], removedTests: string[] = []): MapDiffOutcome =>
|
||||
({ ok: true, changedTests, removedTests, globalTouchfilesChanged: false });
|
||||
|
||||
test('data-file change selects only map-changed tests, reason map-diff', () => {
|
||||
const result = selectTests(
|
||||
[TOUCHFILES_DATA_PATH],
|
||||
E2E_TOUCHFILES,
|
||||
GLOBAL_TOUCHFILES,
|
||||
{ mapDiff: okOutcome(['browse-basic']) },
|
||||
);
|
||||
expect(result.selected).toEqual(['browse-basic']);
|
||||
expect(result.reason).toBe('map-diff');
|
||||
expect(result.skipped.length).toBe(Object.keys(E2E_TOUCHFILES).length - 1);
|
||||
});
|
||||
|
||||
test('map-diff result unions with pattern matching for other changed files', () => {
|
||||
const result = selectTests(
|
||||
[TOUCHFILES_DATA_PATH, 'retro/SKILL.md'],
|
||||
E2E_TOUCHFILES,
|
||||
GLOBAL_TOUCHFILES,
|
||||
{ mapDiff: okOutcome(['browse-basic']) },
|
||||
);
|
||||
expect(result.selected).toContain('browse-basic'); // from map-diff
|
||||
expect(result.selected).toContain('retro'); // from pattern match
|
||||
expect(result.selected).toContain('retro-base-branch');
|
||||
expect(result.selected).not.toContain('cso-full-audit');
|
||||
expect(result.reason).toBe('map-diff');
|
||||
});
|
||||
|
||||
test('changedTests scoped to the map being selected against', () => {
|
||||
// 'judge one' is an LLM-judge key, not an E2E key — must not leak in.
|
||||
const result = selectTests(
|
||||
[TOUCHFILES_DATA_PATH],
|
||||
E2E_TOUCHFILES,
|
||||
GLOBAL_TOUCHFILES,
|
||||
{ mapDiff: okOutcome(['browse-basic', 'judge one']) },
|
||||
);
|
||||
expect(result.selected).toEqual(['browse-basic']);
|
||||
});
|
||||
|
||||
test('removedTests reported, not selected', () => {
|
||||
const result = selectTests(
|
||||
[TOUCHFILES_DATA_PATH],
|
||||
E2E_TOUCHFILES,
|
||||
GLOBAL_TOUCHFILES,
|
||||
{ mapDiff: okOutcome([], ['some-retired-test']) },
|
||||
);
|
||||
expect(result.selected).toEqual([]);
|
||||
expect(result.removedTests).toEqual(['some-retired-test']);
|
||||
});
|
||||
|
||||
test('FAIL-CLOSED: failed map-diff runs all with cause in reason', () => {
|
||||
const result = selectTests(
|
||||
[TOUCHFILES_DATA_PATH],
|
||||
E2E_TOUCHFILES,
|
||||
GLOBAL_TOUCHFILES,
|
||||
{ mapDiff: { ok: false, cause: 'import-failed' } },
|
||||
);
|
||||
expect(result.selected.length).toBe(Object.keys(E2E_TOUCHFILES).length);
|
||||
expect(result.reason).toBe('global — touchfiles-data changed (import-failed)');
|
||||
});
|
||||
|
||||
test('GLOBAL_TOUCHFILES edit inside data file runs all', () => {
|
||||
const result = selectTests(
|
||||
[TOUCHFILES_DATA_PATH],
|
||||
E2E_TOUCHFILES,
|
||||
GLOBAL_TOUCHFILES,
|
||||
{ mapDiff: { ok: true, changedTests: [], removedTests: [], globalTouchfilesChanged: true } },
|
||||
);
|
||||
expect(result.selected.length).toBe(Object.keys(E2E_TOUCHFILES).length);
|
||||
expect(result.reason).toContain('GLOBAL_TOUCHFILES');
|
||||
});
|
||||
|
||||
test('a real global touchfile hit still wins over map-diff', () => {
|
||||
const result = selectTests(
|
||||
[TOUCHFILES_DATA_PATH, 'test/helpers/session-runner.ts'],
|
||||
E2E_TOUCHFILES,
|
||||
GLOBAL_TOUCHFILES,
|
||||
{ mapDiff: okOutcome(['browse-basic']) },
|
||||
);
|
||||
expect(result.selected.length).toBe(Object.keys(E2E_TOUCHFILES).length);
|
||||
expect(result.reason).toBe('global: test/helpers/session-runner.ts');
|
||||
});
|
||||
|
||||
test('no data-file change → classic diff behavior, no map-diff consulted', () => {
|
||||
const result = selectTests(['retro/SKILL.md'], E2E_TOUCHFILES, GLOBAL_TOUCHFILES, {
|
||||
// Poison injection: if the wiring consulted this, the test would fail.
|
||||
mapDiff: { ok: false, cause: 'import-failed' },
|
||||
});
|
||||
expect(result.reason).toBe('diff');
|
||||
expect(result.selected).toContain('retro');
|
||||
expect(result.removedTests).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// --- Layer 3: thin shell against a temp git repo ---
|
||||
|
||||
const OLD_FIXTURE = `export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'alpha': ['a/**'],
|
||||
'beta': ['b/**'],
|
||||
};
|
||||
export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
'alpha': 'gate',
|
||||
'beta': 'periodic',
|
||||
};
|
||||
export const LLM_JUDGE_TOUCHFILES: Record<string, string[]> = {
|
||||
'judge one': ['j/SKILL.md'],
|
||||
};
|
||||
export const GLOBAL_TOUCHFILES = [
|
||||
'test/helpers/session-runner.ts',
|
||||
];
|
||||
`;
|
||||
|
||||
describe('diffTouchfileMaps (git + bun-child shell)', () => {
|
||||
let repo: string;
|
||||
|
||||
const git = (args: string[]) => {
|
||||
const result = spawnSync(
|
||||
'git',
|
||||
['-c', 'user.email=test@test', '-c', 'user.name=test', '-c', 'commit.gpgsign=false', '-c', 'tag.gpgsign=false', ...args],
|
||||
{ cwd: repo, stdio: 'pipe', timeout: 10000 },
|
||||
);
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`git ${args.join(' ')} failed: ${result.stderr?.toString()}`);
|
||||
}
|
||||
};
|
||||
|
||||
const commitDataFile = (source: string, message: string) => {
|
||||
const filePath = path.join(repo, TOUCHFILES_DATA_PATH);
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
fs.writeFileSync(filePath, source);
|
||||
git(['add', TOUCHFILES_DATA_PATH]);
|
||||
git(['commit', '-q', '-m', message]);
|
||||
};
|
||||
|
||||
beforeAll(() => {
|
||||
repo = fs.mkdtempSync(path.join(os.tmpdir(), 'touchfiles-map-diff-repo-'));
|
||||
git(['init', '-q']);
|
||||
commitDataFile(OLD_FIXTURE, 'old maps');
|
||||
git(['tag', 'old-maps']);
|
||||
// Commit with a broken data file (unterminated string → bun import fails)
|
||||
commitDataFile("export const E2E_TOUCHFILES = {\n 'broken: ['\n", 'broken maps');
|
||||
git(['tag', 'broken-maps']);
|
||||
// Commit with wrong shape (tiers value is a number)
|
||||
commitDataFile(
|
||||
'export const E2E_TOUCHFILES = {};\n'
|
||||
+ "export const E2E_TIERS = { 'alpha': 1 };\n"
|
||||
+ 'export const LLM_JUDGE_TOUCHFILES = {};\n'
|
||||
+ 'export const GLOBAL_TOUCHFILES = [];\n',
|
||||
'wrong shape',
|
||||
);
|
||||
git(['tag', 'wrong-shape']);
|
||||
// Commit that deletes the data file entirely (ref exists, file does not)
|
||||
git(['rm', '-q', TOUCHFILES_DATA_PATH]);
|
||||
git(['commit', '-q', '-m', 'file deleted']);
|
||||
git(['tag', 'no-data-file']);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
fs.rmSync(repo, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('happy path: old version from git vs injected new maps', () => {
|
||||
const newMaps: TouchfileMaps = {
|
||||
E2E_TOUCHFILES: { 'alpha': ['a/**'], 'beta': ['b/**', 'new-dep.ts'], 'gamma': ['g/**'] },
|
||||
E2E_TIERS: { 'alpha': 'periodic', 'beta': 'periodic', 'gamma': 'gate' },
|
||||
LLM_JUDGE_TOUCHFILES: { 'judge one': ['j/SKILL.md'] },
|
||||
GLOBAL_TOUCHFILES: ['test/helpers/session-runner.ts'],
|
||||
};
|
||||
const outcome = diffTouchfileMaps('old-maps', repo, newMaps);
|
||||
if (!outcome.ok) throw new Error(`expected ok, got cause=${outcome.cause}`);
|
||||
expect(outcome.changedTests).toEqual(['alpha', 'beta', 'gamma']); // tier flip, dep edit, added
|
||||
expect(outcome.removedTests).toEqual([]);
|
||||
expect(outcome.globalTouchfilesChanged).toBe(false);
|
||||
});
|
||||
|
||||
test('removed key reported from the git version too', () => {
|
||||
const newMaps: TouchfileMaps = {
|
||||
E2E_TOUCHFILES: { 'alpha': ['a/**'] },
|
||||
E2E_TIERS: { 'alpha': 'gate' },
|
||||
LLM_JUDGE_TOUCHFILES: { 'judge one': ['j/SKILL.md'] },
|
||||
GLOBAL_TOUCHFILES: ['test/helpers/session-runner.ts'],
|
||||
};
|
||||
const outcome = diffTouchfileMaps('old-maps', repo, newMaps);
|
||||
if (!outcome.ok) throw new Error(`expected ok, got cause=${outcome.cause}`);
|
||||
expect(outcome.changedTests).toEqual([]);
|
||||
expect(outcome.removedTests).toEqual(['beta']);
|
||||
});
|
||||
|
||||
test('missing base ref → fail-closed with missing-base-ref', () => {
|
||||
const outcome = diffTouchfileMaps('no-such-ref-anywhere', repo);
|
||||
expect(outcome).toEqual({ ok: false, cause: 'missing-base-ref' });
|
||||
});
|
||||
|
||||
test('ref exists but file absent → fail-closed with git-show-failed', () => {
|
||||
const outcome = diffTouchfileMaps('no-data-file', repo);
|
||||
expect(outcome).toEqual({ ok: false, cause: 'git-show-failed' });
|
||||
});
|
||||
|
||||
test('old file fails to import → fail-closed with import-failed', () => {
|
||||
const outcome = diffTouchfileMaps('broken-maps', repo);
|
||||
expect(outcome).toEqual({ ok: false, cause: 'import-failed' });
|
||||
});
|
||||
|
||||
test('old file has unexpected shape → fail-closed with shape-mismatch', () => {
|
||||
const outcome = diffTouchfileMaps('wrong-shape', repo);
|
||||
expect(outcome).toEqual({ ok: false, cause: 'shape-mismatch' });
|
||||
});
|
||||
|
||||
test('end-to-end against this repo: HEAD version evaluates and diffs', () => {
|
||||
// touchfiles-data.ts exists at HEAD (change-set 1). Whatever the working
|
||||
// tree currently holds, the outcome must be a successful evaluation —
|
||||
// assert shape, not content, so this stays green before and after the
|
||||
// change-set commits.
|
||||
const outcome = diffTouchfileMaps('HEAD', ROOT);
|
||||
if (!outcome.ok) throw new Error(`expected ok, got cause=${outcome.cause}`);
|
||||
expect(Array.isArray(outcome.changedTests)).toBe(true);
|
||||
expect(Array.isArray(outcome.removedTests)).toBe(true);
|
||||
expect(typeof outcome.globalTouchfilesChanged).toBe('boolean');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user