mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-07 17:28:23 +01:00
plan-ceo-review gate: hold the final stop until ranked Findings are written (GST-1461) (#4047)
* investigate gate: undo gated commits on branches behind the baseline (GST-1461 fc#1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * plan-ceo-review gate: hold Stop/SubagentStop until ranked Findings are written (GST-1461) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: unit-test rankedRisksGap so lib/ceo-review-gate.ts has a pregate lane (GST-1461) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: plan-ceo-review gate never names or accepts a file inside a skills/ tree (GST-1461) Review-Finding: p1/red-team#1 reader=red-team Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: gstack-skill-gates settings quotes hook paths for the shell (GST-1461 ov#5) Review-Finding: bin/gstack-skill-gates:60:outside-voice reader=outside-voice Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: investigate gate restores raw bytes, index and mode; attributes come from the baseline (GST-1461 ov#1-4) Review-Finding: lib/investigate-gate.ts:126:outside-voice reader=outside-voice Review-Finding: lib/investigate-gate.ts:254:outside-voice reader=outside-voice Review-Finding: lib/investigate-gate.ts:256:outside-voice reader=outside-voice Review-Finding: lib/investigate-gate.ts:64:outside-voice reader=outside-voice Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: investigate gate baseline includes staged-only changes so a rollback keeps them (GST-1461) Review-Finding: p2/outside-voice#1 reader=outside-voice Review-Finding: p2/adversarial-native#6 reader=adversarial-native Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: plan-ceo-review gate judges skills/ paths from the repo root and filters the repo scan too (GST-1461) Review-Finding: p2/outside-voice#2 reader=outside-voice Review-Finding: p2/adversarial-native#2 reader=adversarial-native Review-Finding: p2/adversarial-native#3 reader=adversarial-native Review-Finding: p2/red-team#2 reader=red-team Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: skill-gates restore modes and skills/ exclusion match the code (GST-1461) Review-Finding: p2/adversarial-native#7 reader=adversarial-native Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: changelog entry covers the investigate gate rollback fixes (GST-1461) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * ci: retrigger pull_request checks after stale dirty state Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed
1 parent
9e809ab5b1
commit
4743132ca0
26 files changed
+1166
-172
No files matched your search
+118
-57
@@ -6,6 +6,9 @@
|
||||
* stand-in for the Anthropic Messages API that plays a scripted session, so
|
||||
* no model is called and nothing is spent. Then it checks the GSTACK_GATE
|
||||
* markers in the session record, the hook log and the transcript JSONL.
|
||||
* Two scenarios share the server: the /investigate gate (Edit and sed -i
|
||||
* blocked until a root cause) and the /plan-ceo-review gate (Stop blocked
|
||||
* until the output file ranks the top risks, written through Bash).
|
||||
*
|
||||
* bun scripts/skill-gates-smoke.ts [--claude <path to claude>] [--keep]
|
||||
*
|
||||
@@ -24,11 +27,10 @@ const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const args = process.argv.slice(2);
|
||||
const keep = args.includes('--keep');
|
||||
const work = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-gates-smoke-')));
|
||||
const home = path.join(work, 'home');
|
||||
const repo = path.join(work, 'repo');
|
||||
const state = path.join(work, 'state');
|
||||
const SRC = 'export const total = (a: number, b: number) => a - b;\n';
|
||||
const FIXED = 'export const total = (a: number, b: number) => a + b;\n';
|
||||
const PLAN = '# Plan: one-click checkout\n\nShip one-click checkout behind a flag.\n';
|
||||
const FINDINGS = '\n## Findings\n\n1. Payment retries can double-charge when the provider times out.\n2. No rollback path for the new orders schema.\n';
|
||||
|
||||
function run(cmd: string, argv: string[], cwd: string, timeout = 120_000): string {
|
||||
const r = spawnSync(cmd, argv, { cwd, encoding: 'utf8', timeout });
|
||||
@@ -45,19 +47,73 @@ function claudeBin(): string {
|
||||
return path.join(dir, 'node_modules', '.bin', 'claude');
|
||||
}
|
||||
|
||||
const file = (rel: string) => path.join(repo, rel);
|
||||
/** The scripted session: one assistant turn per main-loop request, chosen by how many assistant turns the request already holds. */
|
||||
const SCRIPT: Array<Array<Record<string, unknown>>> = [
|
||||
[{ type: 'text', text: 'Loading /investigate.' }, { type: 'tool_use', id: 'toolu_s1', name: 'Skill', input: { skill: 'investigate' } }],
|
||||
[{ type: 'tool_use', id: 'toolu_r1', name: 'Read', input: { file_path: file('src/total.ts') } }],
|
||||
[{ type: 'tool_use', id: 'toolu_e1', name: 'Edit', input: { file_path: file('src/total.ts'), old_string: 'a - b', new_string: 'a + b' } }],
|
||||
[{ type: 'tool_use', id: 'toolu_b1', name: 'Bash', input: { command: "sed -i 's/a - b/a + b/' src/total.ts", description: 'Patch through the shell' } }],
|
||||
[{ type: 'text', text: 'Root cause: total() subtracts b instead of adding it (src/total.ts:1).' }, { type: 'tool_use', id: 'toolu_r2', name: 'Read', input: { file_path: file('src/total.ts') } }],
|
||||
[{ type: 'tool_use', id: 'toolu_e2', name: 'Edit', input: { file_path: file('src/total.ts'), old_string: 'a - b', new_string: 'a + b' } }],
|
||||
[{ type: 'text', text: 'Fixed total().' }],
|
||||
type Block = Record<string, unknown>;
|
||||
interface Dirs { home: string; repo: string; state: string }
|
||||
interface Seen { code: number; transcript: string; record: Record<string, any> | null; log: string; requests: string; repo: string }
|
||||
interface Scenario { name: string; hook: string; flag: string; skill: string; prompt: string; seed: (repo: string) => void; script: (repo: string) => Block[][]; checks: (s: Seen) => Array<[string, boolean]> }
|
||||
|
||||
const SCENARIOS: Scenario[] = [
|
||||
{
|
||||
name: 'investigate', hook: 'investigate-gate', flag: '--investigate', skill: 'investigate', prompt: 'total() returns the wrong value; investigate and fix it',
|
||||
seed: repo => { fs.mkdirSync(path.join(repo, 'src'), { recursive: true }); fs.writeFileSync(path.join(repo, 'src/total.ts'), SRC); },
|
||||
script: repo => [
|
||||
[{ type: 'text', text: 'Loading /investigate.' }, { type: 'tool_use', id: 'toolu_s1', name: 'Skill', input: { skill: 'investigate' } }],
|
||||
[{ type: 'tool_use', id: 'toolu_r1', name: 'Read', input: { file_path: path.join(repo, 'src/total.ts') } }],
|
||||
[{ type: 'tool_use', id: 'toolu_e1', name: 'Edit', input: { file_path: path.join(repo, 'src/total.ts'), old_string: 'a - b', new_string: 'a + b' } }],
|
||||
[{ type: 'tool_use', id: 'toolu_b1', name: 'Bash', input: { command: "sed -i 's/a - b/a + b/' src/total.ts", description: 'Patch through the shell' } }],
|
||||
[{ type: 'text', text: 'Root cause: total() subtracts b instead of adding it (src/total.ts:1).' }, { type: 'tool_use', id: 'toolu_r2', name: 'Read', input: { file_path: path.join(repo, 'src/total.ts') } }],
|
||||
[{ type: 'tool_use', id: 'toolu_e2', name: 'Edit', input: { file_path: path.join(repo, 'src/total.ts'), old_string: 'a - b', new_string: 'a + b' } }],
|
||||
[{ type: 'text', text: 'Fixed total().' }],
|
||||
],
|
||||
checks: s => {
|
||||
const run0 = s.record ? Object.values(s.record.hooks['investigate-gate']?.runs ?? {})[0] as Record<string, unknown> | undefined : undefined;
|
||||
return [
|
||||
['claude -p exited 0', s.code === 0],
|
||||
['record: one run, fired', !!run0 && typeof run0.fired_at === 'string'],
|
||||
['record: blocked twice (Edit, then Bash sed -i)', run0?.blocked === 2],
|
||||
['record: released after the Root cause line', typeof run0?.released_at === 'string'],
|
||||
['record: no hook errors', (s.record?.hooks['investigate-gate']?.errors ?? []).length === 0],
|
||||
['log: fired, blocked via=edit, blocked via=bash, released', /event=fired/.test(s.log) && /event=blocked .*via=edit/.test(s.log) && /event=blocked .*via=bash/.test(s.log) && /event=released/.test(s.log)],
|
||||
['transcript: deny reason carries the blocked via=edit marker (tool_result)', /"tool_result"[^\n]*GSTACK_GATE investigate-gate blocked [^"]*via=edit/.test(s.transcript)],
|
||||
['transcript: PostToolUse block carries the blocked via=bash marker (hook_blocking_error)', /hook_blocking_error[^\n]*GSTACK_GATE investigate-gate blocked [^"]*via=bash/.test(s.transcript)],
|
||||
['transcript: fired and released ride hook_system_message', /hook_system_message[^\n]*GSTACK_GATE investigate-gate fired/.test(s.transcript) && /hook_system_message[^\n]*GSTACK_GATE investigate-gate released/.test(s.transcript)],
|
||||
['requests: fired/released systemMessage markers never reach the model', !/investigate-gate (fired|released)/.test(s.requests)],
|
||||
['requests: the model saw both block reasons', /investigate-gate blocked [^"]*via=edit/.test(s.requests) && /investigate-gate blocked [^"]*via=bash/.test(s.requests)],
|
||||
['repo: the sed -i write was put back, then the released Edit applied', fs.readFileSync(path.join(s.repo, 'src/total.ts'), 'utf8') === FIXED],
|
||||
];
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'plan-ceo-review', hook: 'ceo-review-gate', flag: '--ceo-review', skill: 'plan-ceo-review', prompt: 'review plan.md as a CEO and write the review into it',
|
||||
seed: repo => fs.writeFileSync(path.join(repo, 'README.md'), '# smoke\n'),
|
||||
script: repo => [
|
||||
[{ type: 'text', text: 'Loading /plan-ceo-review.' }, { type: 'tool_use', id: 'toolu_c1', name: 'Skill', input: { skill: 'plan-ceo-review' } }],
|
||||
[{ type: 'tool_use', id: 'toolu_w1', name: 'Write', input: { file_path: path.join(repo, 'plan.md'), content: PLAN } }],
|
||||
[{ type: 'text', text: 'Review complete.' }],
|
||||
[{ type: 'tool_use', id: 'toolu_b2', name: 'Bash', input: { command: `cat >> plan.md <<'EOF'\n${FINDINGS}EOF`, description: 'Append the ranked findings' } }],
|
||||
[{ type: 'text', text: 'Findings added to plan.md.' }],
|
||||
],
|
||||
checks: s => {
|
||||
const h = s.record?.hooks['ceo-review-gate'];
|
||||
const run0 = h ? Object.values(h.runs ?? {})[0] as Record<string, unknown> | undefined : undefined;
|
||||
return [
|
||||
['claude -p exited 0', s.code === 0],
|
||||
['record: one run, fired', !!run0 && typeof run0.fired_at === 'string'],
|
||||
['record: blocked once (Stop with no Findings)', run0?.blocked === 1],
|
||||
['record: released by plan.md after the Bash heredoc', typeof run0?.released_at === 'string' && run0?.file === 'plan.md'],
|
||||
['record: no hook errors', (h?.errors ?? []).length === 0],
|
||||
['log: fired, blocked via=stop files=plan.md, released file=plan.md', /event=fired/.test(s.log) && /event=blocked .*via=stop files=plan\.md n=1/.test(s.log) && /event=released .*file=plan\.md/.test(s.log)],
|
||||
['transcript: the Stop block carries the blocked via=stop marker (hook_blocking_error)', /hook_blocking_error[^\n]*GSTACK_GATE ceo-review-gate blocked [^"]*via=stop/.test(s.transcript)],
|
||||
['transcript: fired and released ride hook_system_message', /hook_system_message[^\n]*GSTACK_GATE ceo-review-gate fired/.test(s.transcript) && /hook_system_message[^\n]*GSTACK_GATE ceo-review-gate released/.test(s.transcript)],
|
||||
['requests: fired/released systemMessage markers never reach the model', !/ceo-review-gate (fired|released)/.test(s.requests)],
|
||||
['requests: the model saw the block reason and where to write', /ceo-review-gate blocked [^"]*via=stop/.test(s.requests) && s.requests.includes("a '## Findings' section ranking the top risks")],
|
||||
['repo: plan.md ends with the ranked Findings', fs.readFileSync(path.join(s.repo, 'plan.md'), 'utf8') === PLAN + FINDINGS],
|
||||
];
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
function sse(content: Array<Record<string, unknown>>, model: string): string {
|
||||
function sse(content: Block[], model: string): string {
|
||||
const ev = (name: string, data: unknown) => `event: ${name}\ndata: ${JSON.stringify(data)}\n\n`;
|
||||
let out = ev('message_start', { type: 'message_start', message: { id: `msg_smoke_${Math.random().toString(36).slice(2)}`, type: 'message', role: 'assistant', model, content: [], stop_reason: null, stop_sequence: null, usage: { input_tokens: 1, output_tokens: 1 } } });
|
||||
content.forEach((b, i) => {
|
||||
@@ -71,18 +127,21 @@ function sse(content: Array<Record<string, unknown>>, model: string): string {
|
||||
return out + ev('message_stop', { type: 'message_stop' });
|
||||
}
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const claude = claudeBin();
|
||||
const version = run(claude, ['--version'], work).trim();
|
||||
fs.mkdirSync(path.join(home, '.claude', 'skills', 'investigate'), { recursive: true });
|
||||
fs.copyFileSync(path.join(ROOT, 'investigate', 'SKILL.md'), path.join(home, '.claude', 'skills', 'investigate', 'SKILL.md'));
|
||||
fs.mkdirSync(path.join(repo, 'src'), { recursive: true });
|
||||
fs.writeFileSync(file('src/total.ts'), SRC);
|
||||
run('git', ['init', '-q', '-b', 'main'], repo);
|
||||
run('git', ['add', '-A'], repo);
|
||||
run('git', ['-c', 'user.name=Smoke fixture', '-c', 'user.email=smoke@gstack.test', 'commit', '-q', '-m', 'seed'], repo);
|
||||
const settingsFile = path.join(work, 'settings.json');
|
||||
fs.writeFileSync(settingsFile, run(process.execPath, ['--no-env-file', '--config=/dev/null', path.join(ROOT, 'bin', 'gstack-skill-gates'), 'settings', '--investigate'], work));
|
||||
const readIf = (file: string) => (fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : '');
|
||||
|
||||
/** One scenario: its own HOME, repo and state root, one claude -p run against the scripted server. */
|
||||
async function play(claude: string, sc: Scenario): Promise<[number, number]> {
|
||||
const dirs: Dirs = { home: path.join(work, sc.name, 'home'), repo: path.join(work, sc.name, 'repo'), state: path.join(work, sc.name, 'state') };
|
||||
fs.mkdirSync(path.join(dirs.home, '.claude', 'skills', sc.skill), { recursive: true });
|
||||
fs.copyFileSync(path.join(ROOT, sc.skill, 'SKILL.md'), path.join(dirs.home, '.claude', 'skills', sc.skill, 'SKILL.md'));
|
||||
fs.mkdirSync(dirs.repo, { recursive: true });
|
||||
sc.seed(dirs.repo);
|
||||
run('git', ['init', '-q', '-b', 'main'], dirs.repo);
|
||||
run('git', ['add', '-A'], dirs.repo);
|
||||
run('git', ['-c', 'user.name=Smoke fixture', '-c', 'user.email=smoke@gstack.test', 'commit', '-q', '-m', 'seed'], dirs.repo);
|
||||
const settingsFile = path.join(work, sc.name, 'settings.json');
|
||||
fs.writeFileSync(settingsFile, run(process.execPath, ['--no-env-file', '--config=/dev/null', path.join(ROOT, 'bin', 'gstack-skill-gates'), 'settings', sc.flag], work));
|
||||
const script = sc.script(dirs.repo);
|
||||
|
||||
const requests: string[] = [];
|
||||
const server = Bun.serve({ port: 0, hostname: '127.0.0.1', async fetch(req) {
|
||||
@@ -91,52 +150,54 @@ async function main(): Promise<number> {
|
||||
if (!new URL(req.url).pathname.endsWith('/messages')) return Response.json({ input_tokens: 1 });
|
||||
const body = JSON.parse(raw || '{}');
|
||||
const main = Array.isArray(body.tools) && body.tools.length > 0;
|
||||
const content = main ? (SCRIPT[body.messages.filter((m: { role: string }) => m.role === 'assistant').length] ?? [{ type: 'text', text: 'Done.' }]) : [{ type: 'text', text: 'ok' }];
|
||||
const content = main ? (script[body.messages.filter((m: { role: string }) => m.role === 'assistant').length] ?? [{ type: 'text', text: 'Done.' }]) : [{ type: 'text', text: 'ok' }];
|
||||
if (!body.stream) return Response.json({ id: 'msg_smoke', type: 'message', role: 'assistant', model: body.model, content, stop_reason: 'end_turn', usage: { input_tokens: 1, output_tokens: 1 } });
|
||||
return new Response(sse(content, body.model), { headers: { 'content-type': 'text/event-stream' } });
|
||||
} });
|
||||
|
||||
const env = {
|
||||
PATH: `${path.dirname(process.execPath)}:${process.env.PATH ?? ''}`, HOME: home, GSTACK_STATE_ROOT: state,
|
||||
PATH: `${path.dirname(process.execPath)}:${process.env.PATH ?? ''}`, HOME: dirs.home, GSTACK_STATE_ROOT: dirs.state,
|
||||
ANTHROPIC_BASE_URL: `http://127.0.0.1:${server.port}`, ANTHROPIC_API_KEY: 'sk-smoke-not-a-key',
|
||||
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1', DISABLE_AUTOUPDATER: '1',
|
||||
};
|
||||
const proc = Bun.spawn([claude, '-p', 'total() returns the wrong value; investigate and fix it', '--settings', settingsFile, '--permission-mode', 'bypassPermissions', '--output-format', 'json', '--max-turns', '20'], { cwd: repo, env, stdout: 'pipe', stderr: 'pipe' });
|
||||
const proc = Bun.spawn([claude, '-p', sc.prompt, '--settings', settingsFile, '--permission-mode', 'bypassPermissions', '--output-format', 'json', '--max-turns', '20'], { cwd: dirs.repo, env, stdout: 'pipe', stderr: 'pipe' });
|
||||
const killer = setTimeout(() => proc.kill(), 180_000);
|
||||
const [out, err, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
|
||||
clearTimeout(killer);
|
||||
server.stop(true);
|
||||
|
||||
const transcripts = fs.existsSync(path.join(home, '.claude', 'projects')) ? fs.readdirSync(path.join(home, '.claude', 'projects'), { recursive: true }).map(String).filter(f => f.endsWith('.jsonl')) : [];
|
||||
const transcript = transcripts.length ? fs.readFileSync(path.join(home, '.claude', 'projects', transcripts[0]), 'utf8') : '';
|
||||
const records = fs.existsSync(path.join(state, 'skill-gates')) ? fs.readdirSync(path.join(state, 'skill-gates')) : [];
|
||||
const record = records.length ? JSON.parse(fs.readFileSync(path.join(state, 'skill-gates', records[0], 'record.json'), 'utf8')) : null;
|
||||
const run0 = record ? Object.values(record.hooks['investigate-gate']?.runs ?? {})[0] as Record<string, unknown> | undefined : undefined;
|
||||
const log = fs.existsSync(path.join(state, 'hooks', 'investigate-gate.log')) ? fs.readFileSync(path.join(state, 'hooks', 'investigate-gate.log'), 'utf8') : '';
|
||||
const markerLines = [...new Set(transcript.match(/GSTACK_GATE investigate-gate [a-z]+ [^"\\]*/g) ?? [])];
|
||||
const later = requests.join('\n');
|
||||
|
||||
const checks: Array<[string, boolean]> = [
|
||||
['claude -p exited 0', code === 0],
|
||||
['record: one run, fired', !!run0 && typeof run0.fired_at === 'string'],
|
||||
['record: blocked twice (Edit, then Bash sed -i)', run0?.blocked === 2],
|
||||
['record: released after the Root cause line', typeof run0?.released_at === 'string'],
|
||||
['record: no hook errors', (record?.hooks['investigate-gate']?.errors ?? []).length === 0],
|
||||
['log: fired, blocked via=edit, blocked via=bash, released', /event=fired/.test(log) && /event=blocked .*via=edit/.test(log) && /event=blocked .*via=bash/.test(log) && /event=released/.test(log)],
|
||||
['transcript: deny reason carries the blocked via=edit marker (tool_result)', /"tool_result"[^\n]*GSTACK_GATE investigate-gate blocked [^"]*via=edit/.test(transcript)],
|
||||
['transcript: PostToolUse block carries the blocked via=bash marker (hook_blocking_error)', /hook_blocking_error[^\n]*GSTACK_GATE investigate-gate blocked [^"]*via=bash/.test(transcript)],
|
||||
['transcript: fired and released ride hook_system_message', /hook_system_message[^\n]*GSTACK_GATE investigate-gate fired/.test(transcript) && /hook_system_message[^\n]*GSTACK_GATE investigate-gate released/.test(transcript)],
|
||||
['requests: fired/released systemMessage markers never reach the model', !/investigate-gate (fired|released)/.test(later)],
|
||||
['requests: the model saw both block reasons', /investigate-gate blocked [^"]*via=edit/.test(later) && /investigate-gate blocked [^"]*via=bash/.test(later)],
|
||||
['repo: the sed -i write was put back, then the released Edit applied', fs.readFileSync(file('src/total.ts'), 'utf8') === FIXED],
|
||||
];
|
||||
process.stdout.write(`SMOKE: claude ${version} (fake Messages API, ${requests.length} requests, $0)\n`);
|
||||
for (const line of markerLines) process.stdout.write(` transcript marker: ${line}\n`);
|
||||
for (const [name, ok] of checks) process.stdout.write(`${ok ? 'PASS' : 'FAIL'} ${name}\n`);
|
||||
const projects = path.join(dirs.home, '.claude', 'projects');
|
||||
const transcripts = fs.existsSync(projects) ? fs.readdirSync(projects, { recursive: true }).map(String).filter(f => f.endsWith('.jsonl')) : [];
|
||||
const records = fs.existsSync(path.join(dirs.state, 'skill-gates')) ? fs.readdirSync(path.join(dirs.state, 'skill-gates')) : [];
|
||||
const seen: Seen = {
|
||||
code, repo: dirs.repo, requests: requests.join('\n'),
|
||||
transcript: transcripts.length ? readIf(path.join(projects, transcripts[0])) : '',
|
||||
record: records.length ? JSON.parse(readIf(path.join(dirs.state, 'skill-gates', records[0], 'record.json'))) : null,
|
||||
log: readIf(path.join(dirs.state, 'hooks', `${sc.hook}.log`)),
|
||||
};
|
||||
const checks = sc.checks(seen);
|
||||
process.stdout.write(`SCENARIO: ${sc.name} (${requests.length} requests)\n`);
|
||||
for (const line of new Set(seen.transcript.match(new RegExp(`GSTACK_GATE ${sc.hook} [a-z]+ [^"\\\\]*`, 'g')) ?? [])) process.stdout.write(` transcript marker: ${line}\n`);
|
||||
for (const [name, ok] of checks) process.stdout.write(`${ok ? 'PASS' : 'FAIL'} ${sc.name}: ${name}\n`);
|
||||
const failed = checks.filter(([, ok]) => !ok).length;
|
||||
if (failed) process.stdout.write(`claude stdout: ${out.slice(0, 2000)}\nclaude stderr: ${err.slice(0, 2000)}\nwork dir kept: ${work}\n`);
|
||||
if (failed) process.stdout.write(`claude stdout: ${out.slice(0, 2000)}\nclaude stderr: ${err.slice(0, 2000)}\n`);
|
||||
return [failed, checks.length];
|
||||
}
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const claude = claudeBin();
|
||||
const version = run(claude, ['--version'], work).trim();
|
||||
process.stdout.write(`SMOKE: claude ${version} (fake Messages API, $0)\n`);
|
||||
let failed = 0;
|
||||
let total = 0;
|
||||
for (const sc of SCENARIOS) {
|
||||
const [f, n] = await play(claude, sc);
|
||||
failed += f;
|
||||
total += n;
|
||||
}
|
||||
if (failed) process.stdout.write(`work dir kept: ${work}\n`);
|
||||
else if (!keep) fs.rmSync(work, { recursive: true, force: true });
|
||||
process.stdout.write(`SMOKE_RESULT: ${failed ? 'fail' : 'pass'} checks=${checks.length} failed=${failed}\n`);
|
||||
process.stdout.write(`SMOKE_RESULT: ${failed ? 'fail' : 'pass'} scenarios=${SCENARIOS.length} checks=${total} failed=${failed}\n`);
|
||||
return failed ? 1 : 0;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user