mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 09:56:57 +02:00
garrytan
17ee2e5428
review-army-perf-n-plus-one timed out in 7 of 13 CI runs on this branch (passing 245-280 s of 300). Each session spent ~95 s on setup (the full extracted SKILL, checklist, section greps, exploratory.md, diff-scope/stats/learnings, tooling checks), ran Step 4's core pass, a search-before-recommending WebSearch, and wrote a 10-16 KB report (~100 s after the Red Team returned). The fixture now stages only review/sections/review-army.md plus the performance and red-team checklists, and hands the session the recorded detect-scope, specialist-stats and learnings outputs and the diff. The caller passes --performance (every CI parent already treated the prompt as that force flag against the <50-line skip), declares the core pass, QA, adversarial review, web research, Fix-First and persistence out of scope, and caps the report at the selection line, the SPECIALIST REVIEW block and the Red Team result (30 lines). The Performance specialist and the conditional Red Team are still real foreground subagents, and the report still has to surface the N+1. New assertion: a foreground Performance specialist dispatch precedes the Red Team dispatch. Free controls omit the Performance dispatch or background it, and both fail; the budget lifecycle adapter supplies the current result shape. Touchfiles now include the .rb fixture the case reads.
63 lines
3.9 KiB
TypeScript
63 lines
3.9 KiB
TypeScript
import {expect, test} from 'bun:test';
|
|
import * as fs from 'node:fs';
|
|
import * as os from 'node:os';
|
|
import * as path from 'node:path';
|
|
import {spawnSync} from 'node:child_process';
|
|
const ROOT = path.resolve(import.meta.dir, '..');
|
|
test.each(['complete-control', 'captured-omission', 'claimed-only', 'background', 'missing-report', 'unrelated-report', 'captured-timeout', 'performance-omitted', 'performance-background'])
|
|
('N+1 registered completion contract: %s', scenario => {
|
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'n1-contract-'));
|
|
const facts = path.join(dir, 'facts.json');
|
|
const script = path.join(dir, 'callback.test.ts');
|
|
fs.writeFileSync(script, `
|
|
import {describe,expect,mock,test} from 'bun:test';
|
|
import * as fs from 'node:fs';
|
|
import * as path from 'node:path';
|
|
const root=${JSON.stringify(ROOT)},scenario=${JSON.stringify(scenario)},facts=${JSON.stringify(facts)};
|
|
const actual=await import(path.join(root,'test/helpers/session-runner.ts'));
|
|
const fixture=JSON.parse(fs.readFileSync(path.join(root,'test/fixtures/review-n-plus-one-dispatch.json'),'utf8'));
|
|
const data=structuredClone(['captured-omission','claimed-only'].includes(scenario)?fixture.omission:fixture.ci);
|
|
if(scenario==='background')data.events[1].message.content[0].input.run_in_background=true;
|
|
if(scenario==='performance-background')data.events[0].message.content[0].input.run_in_background=true;
|
|
if(scenario==='performance-omitted')data.events.shift();
|
|
const parsed=actual.parseNDJSON(data.events.map(e=>JSON.stringify(e)));
|
|
let prompt='';
|
|
mock.module(path.join(root,'test/helpers/e2e-helpers.ts'),()=>({
|
|
ROOT:root,runId:'n1-contract',
|
|
describeIfSelected:(name,ids,body)=>{if(ids.includes('review-army-perf-n-plus-one'))describe(name,body);},
|
|
testConcurrentIfSelected:(id,body,timeout)=>{if(id==='review-army-perf-n-plus-one')test(id,body,timeout);},
|
|
logCost(){},createEvalCollector:()=>null,finalizeEvalCollector:async()=>{},
|
|
recordE2E:(_collector,_name,_suite,result,extra)=>fs.writeFileSync(facts,JSON.stringify({
|
|
passed:extra?.passed??result.exitReason==='success',exitReason:result.exitReason,toolCalls:result.toolCalls,prompt,
|
|
})),
|
|
}));
|
|
mock.module(path.join(root,'test/helpers/session-runner.ts'),()=>({...actual,runSkillTest:async options=>{
|
|
expect(options.timeout).toBe(300000);expect(options.maxTurns).toBe(20);
|
|
prompt=options.prompt;
|
|
if(scenario!=='missing-report')fs.writeFileSync(path.join(options.workingDirectory,'review-output.md'),
|
|
scenario==='unrelated-report'?'No relevant evidence':'N+1 queries at posts_controller.rb:7 and :9.'+
|
|
(scenario==='claimed-only'?' Red Team completed.':''));
|
|
return {exitReason:scenario==='captured-timeout'?'timeout':'success',browseErrors:[],toolCalls:parsed.toolCalls,
|
|
transcript:parsed.transcript,output:data.publicAcknowledgement??'Synthetic successful completion control.'};
|
|
}}));
|
|
await import(path.join(root,'test/skill-e2e-review-army.test.ts'));
|
|
`);
|
|
try {
|
|
const child = spawnSync(process.execPath, ['test', script], {
|
|
cwd: dir, encoding: 'utf8', timeout: 10_000,
|
|
env: {PATH: process.env.PATH ?? '', HOME: dir, TMPDIR: dir, NO_COLOR: '1'},
|
|
});
|
|
expect(child.status, child.stdout + child.stderr).toBe(scenario === 'complete-control' ? 0 : 1);
|
|
expect(child.stderr).not.toContain('Unhandled error between tests');
|
|
expect(fs.existsSync(facts), child.stdout + child.stderr).toBe(true);
|
|
const observed = JSON.parse(fs.readFileSync(facts, 'utf8'));
|
|
expect(observed.passed).toBe(scenario === 'complete-control');
|
|
expect(observed.exitReason).toBe(scenario === 'captured-timeout' ? 'timeout' : 'success');
|
|
expect(observed.prompt).toContain('conditional Red Team dispatch');
|
|
expect(observed.prompt).toContain('separate foreground Red Team subagent');
|
|
expect(observed.prompt).toContain('brief acknowledgement');
|
|
} finally {
|
|
fs.rmSync(dir, {recursive: true, force: true});
|
|
}
|
|
});
|