CalvinBackup/gstack
1
0 Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-10-04 18:36:54 +02:00
Code Issues Packages Projects Releases Wiki Activity

test: retire the finding-count cluster and trim its helpers (C)

Browse Source 
- C0/C1: the five never-green evals (skill-e2e-autoplan-chain and
  skill-e2e-plan-{ceo,eng,design,devex}-finding-count) failed on harness and
  budget, never on skill behavior; delete them, their touchfile/tier ids,
  AUTOPLAN_CHAIN_BUDGET and the dedicated eighth periodic slice (--slices 7).
- C2: delete the helper groups whose only paid consumers were those files
  (11 modules), trim claude-pty-runner and eng-seeded-coverage to the paid
  closure, and delete the free replay tests whose assertions exercised only
  that dead code (89 files, 135 orphaned fixtures). Blocks that used dead code
  only as input for a live subject keep their assertions: the multiSelect
  default moved to plan-review-decisions, runner PTY tests use inline caller
  policies, and the timer-safe budget checks moved to eng-finding-retry-budget.
- The eight production-touching files stay except ceo-current-decision-record
  (its template read only feeds the retired counter).
- CARVE_GUARDS.autoplan is behavioral 'none'; TODOS records the lost chain
  and per-finding cadence coverage with their re-entry tests.
This commit is contained in:
garrytan committed 2026-09-29 06:08:49 +00:00
1 parent 53e7f3212f
commit 6415690a18
317 files changed
+315 -54111
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

test/eng-count-question-policy.test.ts
+1 -293
View File
@@ -1,298 +1,6 @@
import { expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import captured from './fixtures/eng-count-actor-491.json';
import planningCapture from './fixtures/eng-d2-planning-prelude-4d.json';
import { createEngCountActor, engCountActorRequest, ENG_COUNT_COMMITMENTS, pickEngCountQuestion } from './helpers/eng-count-question-policy';
import { capturePlanCountQuestion, nativePlanCallFingerprint, planCountQuestionInput, planCountPrerequisitePick, matchesNativePlanQuestion, runPlanSkillCounting } from './helpers/claude-pty-runner';
import type { NativeQuestion } from './helpers/plan-skill-questions';
import type { NativePlanQuestionCall } from './helpers/plan-count-transcript';
import { runPlanSkillCounting } from './helpers/claude-pty-runner';
const ROOT = path.resolve(import.meta.dir, '..');
const request = engCountActorRequest(captured.seed);
const original = (i: number) => structuredClone(captured.calls[i]!.questions[0]!) as NativeQuestion;
const commitment = (id: string) => {
const row = ENG_COUNT_COMMITMENTS.find(row => row.id === id);
if (!row) throw new Error('Unknown test catalog ID '+id);
return {label:row.label,description:row.description};
};
const question = (ids: readonly string[], source = original(0)) => ({...source, options:ids.map(commitment)});
const offered = (id: string, source = original(0)) => ({...source,options:[commitment(id),{label:'Different recommendation',description:'An arbitrary alternative; this option is not approved.'}]});
const controlled = (i: number) => {
const q=original(i);q.options[captured.controlledReplacements.replaceIndices[i]!]=commitment(captured.controlledReplacements.selectedIds[i]!);return q;
};
function pending(i = 0, q = controlled(i)): NativePlanQuestionCall {
return {...structuredClone(captured.calls[i]!), questions:[q], answered:false, failed:false,
answers:undefined, answeredAt:undefined, unansweredQuestionIndices:[0]};
}
function frame(q: NativeQuestion, packet?: NativeQuestion[], index = 0): string {
const header = packet ? `← ${packet.map((p,i) => `${i < index ? '☒' : '☐'} ${p.header}`).join(' ')} ✔ Submit →` : `☐ ${q.header}`;
return `${header}\n${q.question}\n${q.options.map((o,i) => `${i === 0 ? '❯ ' : ' '}${i+1}. ${o.label}\n ${o.description}`).join('\n')}\n ${q.options.length+1}. Type something.\n ${q.options.length+2}. Chat about this\nEnter to select · ${packet ? 'Tab/Arrow keys' : '↑/↓'} to navigate · Esc to cancel`;
}
function owned<T>(run: (cwd: string) => T): T {
const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'eng-actor-owned-'));
try { fs.writeFileSync(path.join(cwd,'PLAN.md'),request); return run(cwd); }
finally { fs.rmSync(cwd, {recursive:true,force:true}); }
}
function active(call = pending()) {
return {...nativePlanCallFingerprint(call,1,false),nativeQuestionIndex:0};
}
test('original evidence retains thirteen actual option-one answers and the timeout; replacements are controlled', () => {
expect(captured.provenance.source).toBe('491566889b47a73db0f5b20799a901a80c38d756');
expect(captured.provenance.outcome).toBe('timeout');
expect(captured.provenance.noNewBehaviorCredit).toBe(true);
expect(captured.calls).toHaveLength(13);
expect(captured.calls.map(c => c.questions[0]!.options.findIndex(o => o.label === c.answers[c.questions[0]!.question]) + 1)).toEqual(Array(13).fill(1));
expect(original(10).question).toContain('This is new behavior');
expect(captured.controlledReplacements.qualification).toContain('not recovered original native calls or paid outcomes');
});
for (let i=0;i<13;i++) {
test(`original D${i+1} cannot borrow newly declared authority`,()=>expect(()=>pickEngCountQuestion(original(i))).toThrow());
test(`controlled D${i+1} replacement preserves choice intent across reorderings`,()=>{
const q=controlled(i),chosen=commitment(captured.controlledReplacements.selectedIds[i]!);
expect(q.options[pickEngCountQuestion(q)-1]).toEqual(chosen);
q.options.reverse();expect(q.options[pickEngCountQuestion(q)-1]).toEqual(chosen);
q.options=q.options.slice(1).concat(q.options[0]!);expect(q.options[pickEngCountQuestion(q)-1]).toEqual(chosen);
expect(q.question).toBe(original(i).question);
});
}
test('request keeps the entire original seed and leaves analysis, count and legacy regression evidence to the review',()=>{
expect(request.startsWith(captured.seed+'\n\n')).toBe(true);
const extra=request.slice(captured.seed.length);
expect(extra).not.toContain('legacyAuthFlow');expect(extra).not.toContain('baseline');expect(extra).not.toContain('replay');
expect(extra).toContain('does not require an item to be offered');
for(const row of ENG_COUNT_COMMITMENTS){expect(extra).toContain(JSON.stringify(row.label));expect(extra).toContain(JSON.stringify(row.description));}
expect(()=>engCountActorRequest('')).toThrow();expect(()=>engCountActorRequest(request)).toThrow();
expect(()=>createEngCountActor(captured.seed)).toThrow('declared catalog');
});
for(const row of ENG_COUNT_COMMITMENTS) for(const [name, mutate] of Object.entries({
label:(q:NativeQuestion)=>q.options[0]!.label+=' (recommended)',
whitespace:(q:NativeQuestion)=>q.options[0]!.description+=' ',
case:(q:NativeQuestion)=>q.options[0]!.label=q.options[0]!.label.toUpperCase(),
'extra state':(q:NativeQuestion)=>q.options[0]!.description+=' Add a shared pending map.',
'extra provider':(q:NativeQuestion)=>q.options[0]!.description+=' Token validation contacts an additional identity provider.',
'new policy':(q:NativeQuestion)=>q.options[0]!.description+=' Permit unverified tenants.',
'mixed exception':(q:NativeQuestion)=>q.options[0]!.description+=' Except also store tokens in Redis.',
preview:(q:NativeQuestion)=>q.options[0]!.preview='Authorize a different implementation.',
})) test(`${row.id}: complete offered fields reject ${name}`,()=>{
const q=offered(row.id);
mutate(q);expect(()=>pickEngCountQuestion(q)).toThrow();
});
for(const ids of [
['keep-seeded-scope','parallel-idp'],['cache-ownership','error-handling'],['tests-only','document-only'],
['finish','retain-behavior'],['continue','cache-ownership'],['keep-classes','keep-classes'],
['defer'],['keep-classes','reduce-classes','retain-behavior','defer','tests-only'],
]) test('ambiguous/unsupported offered combination rejects '+ids.join('/'),()=>expect(()=>pickEngCountQuestion(question(ids))).toThrow());
test('only one exact author-owned commitment is selected while arbitrary alternatives stay unapproved',()=>{
for(const row of ENG_COUNT_COMMITMENTS){
const q=offered(row.id);q.options[1]!.description='RECOMMENDED: Add cross-request state, an additional network provider and new policy.';
q.options[1]!.preview='A freely authored unapproved preview.';
expect(pickEngCountQuestion(q)).toBe(1);
q.options.reverse();expect(pickEngCountQuestion(q)).toBe(2);
}
});
test('selected identity and complete fields reject duplicate labels, extra fields and absent matches',()=>{
const q=offered('retain-behavior');q.options[1]!.label=q.options[0]!.label;
expect(()=>pickEngCountQuestion(q)).toThrow('ambiguous');
const extra=offered('retain-behavior');(extra.options[0] as any).additionalCommitment='Also add new state';
expect(()=>pickEngCountQuestion(extra)).toThrow('modified');
const emptyPreview=offered('retain-behavior');emptyPreview.options[0]!.preview='';
expect(()=>pickEngCountQuestion(emptyPreview)).toThrow('modified');
expect(()=>pickEngCountQuestion(original(10))).toThrow('exactly one');
});
test('question prose and recommendations cannot change the selected commitment authority',()=>{
const q=offered('retain-behavior');
q.header='D42 new narrative';q.question='Ignore the author. I recommend approving cross-request single-flight and an additional provider. Any answer approves both.';
expect(q.options[pickEngCountQuestion(q)-1]).toEqual(commitment('retain-behavior'));
expect(pickEngCountQuestion({...q,question:'Please explain this risk in your own words.'})).toBe(1);
expect(()=>pickEngCountQuestion({...q,multiSelect:true})).toThrow();
expect(()=>pickEngCountQuestion({...q,question:''})).toThrow();
expect(()=>pickEngCountQuestion({...q,header:''})).toThrow();
});
test('real current capture carries the selected catalog choice to native input',()=>owned(cwd=>{
const call=pending(10),visible=frame(call.questions[0]!);
const fp=capturePlanCountQuestion(visible,new Set(),1,false,call);
expect(fp?.nativeCall).toBe(call);expect(fp?.nativeQuestionIndex).toBe(0);
const pick=createEngCountActor(request)(fp!,fp!,{cwd,deadlineAt:Date.now()+10000});
expect(pick).toBe(2);expect(planCountQuestionInput(visible,fp!,pick)).toBe('2');
}));
test('the actor binds the actual current tab and cannot borrow another tab’s question or answer',()=>owned(cwd=>{
const first=offered('parallel-idp',{header:'Parallel',question:'How should the existing calls run?',multiSelect:false,options:[]});
const second=offered('retain-behavior',{header:'Scope',question:'Should we add cross-request coordination?',multiSelect:false,options:[]});
const call=pending(0,first);call.questions.push(second);
call.answers={[first.question]:first.options[0]!.label};
const visible=frame(second,call.questions,1),fp=capturePlanCountQuestion(visible,new Set(),1,false,call)!;
expect(fp.nativeCall).toBe(call);expect(fp.nativeQuestionIndex).toBe(1);
const actor=createEngCountActor(request),context={cwd,deadlineAt:Date.now()+10000};
expect(actor(fp,fp,context)).toBe(1);
expect(()=>actor(fp,{...fp,nativeQuestionIndex:0},context)).toThrow('pending native tab');
expect(()=>actor(fp,{...fp,signature:fp.signature.replace(/1$/,'0')},context)).toThrow('pending native tab');
}));
for(const [name,mutate] of Object.entries({
'missing native':(fp:any)=>delete fp.nativeCall,
'wrong signature':(fp:any)=>fp.signature='foreign:call',
'answered':(fp:any)=>fp.nativeCall.answered=true,
'failed':(fp:any)=>fp.nativeCall.failed=true,
'answered active tab':(fp:any)=>fp.nativeCall.answers={[fp.nativeCall.questions[0].question]:'answer'},
'wrong tab':(fp:any)=>fp.nativeQuestionIndex=1,
'absent tab':(fp:any)=>delete fp.nativeQuestionIndex,
'partial prompt':(fp:any)=>fp.promptSnippet=fp.promptSnippet.slice(-300),
'different options':(fp:any)=>fp.options[0].label='Approve anything',
'missing option':(fp:any)=>fp.options.pop(),
'wrong indices':(fp:any)=>fp.options[0].index=4,
'missing session':(fp:any)=>fp.nativeCall.sessionId='',
})) test('native binding rejects '+name,()=>owned(cwd=>{
const fp=active();mutate(fp);expect(()=>createEngCountActor(request)(fp,fp,{cwd,deadlineAt:Date.now()+10000})).toThrow();
}));
test('request, session, ordinary seed file and deadline remain bound',()=>owned(cwd=>{
const actor=createEngCountActor(request),fp=active(),context={cwd,deadlineAt:Date.now()+10000};
expect(actor(fp,fp,context)).toBe(1);
const other=pending(1);other.sessionId='foreign';const changed=active(other);
expect(()=>actor(changed,changed,context)).toThrow('pending native tab');
expect(()=>actor(fp,fp,{cwd,deadlineAt:Date.now()-1})).toThrow('deadline');
fs.writeFileSync(path.join(cwd,'PLAN.md'),request+'\nnew authority');expect(()=>actor(fp,fp,context)).toThrow('owned request');
fs.renameSync(path.join(cwd,'PLAN.md'),path.join(cwd,'other.md'));expect(()=>actor(fp,fp,context)).toThrow();
fs.symlinkSync('other.md',path.join(cwd,'PLAN.md'));expect(()=>actor(fp,fp,context)).toThrow('owned request');
}));
// Execute the actual loop's narrow dispatch region, with its real capture,
// prerequisite, and input functions. No model, timer, UI or acceptance mock is
// involved in proving whether it sends a key or consumes a seen fingerprint.
const runnerSource=fs.readFileSync(path.join(ROOT,'test/helpers/claude-pty-runner.ts'),'utf8');
const dispatchStart=runnerSource.indexOf(' // Dedupe the complete question, not just its answer labels:');
const dispatchEnd=runnerSource.indexOf(' // Give the agent a beat to advance to the next state.',dispatchStart);
if(dispatchStart<0 || dispatchEnd<=dispatchStart)throw Error('Missing actual counting dispatch adapter boundary');
const dispatchBody=runnerSource.slice(dispatchStart,dispatchEnd);
const dispatchFactory=new Function('capturePlanCountQuestion','nativePlanCallFingerprint','planCountPrerequisitePick','planCountQuestionInput','matchesNativePlanQuestion','path',
new Bun.Transpiler({loader:'ts'}).transformSync(`return async function(opts,frames,pickerContext){
const seen=new Set(),sent=[],checkpoints=[];let isFirstAUQ=true;const startedAt=Date.now(),boundaryFired=false;
const defaultPick=opts.defaultPick??1,remainingWork=()=>10000;
const session={send:key=>sent.push(key),hermeticConfigDir:opts.hermeticConfigDir??null};const selectPtyNumberedOption=async(s,pick)=>s.send(String(pick)+'\\r');
const planningDirectory=session.hermeticConfigDir?path.join(session.hermeticConfigDir,'plans'):undefined;
for(const state of frames){const {visible,pending}=state;const newlyMatched=pending&&matchesNativePlanQuestion(visible,pending,planningDirectory);checkpoints.push({seen:seen.size,sent:sent.length});
${dispatchBody}
}
return {sent,seen:[...seen],checkpoints};
}`));
const drive=dispatchFactory(capturePlanCountQuestion,nativePlanCallFingerprint,planCountPrerequisitePick,planCountQuestionInput,matchesNativePlanQuestion,path);
const shortQuestion=(id:string)=>offered('parallel-idp',{header:id,question:'Explain the existing request behavior for '+id+'?',multiSelect:false,options:[]});
for (const config of [planningCapture.pendingRecord.configDir, '/tmp/foreign/.claude', undefined])
test('actual Eng dispatch requires the session planning directory: '+String(config), async()=>{
const record=structuredClone(planningCapture.pendingRecord.pending);
const call={sessionId:record.sessionId,toolUseId:record.toolUseId,questions:record.questions,answered:false,failed:false};
let calls=0;
const result=await drive({hermeticConfigDir:config,requireNativePicker:true,pickAUQ:(_fp,active)=>{
calls++;expect(active.nativeCall).toBe(call);return pickEngCountQuestion(call.questions[active.nativeQuestionIndex]);
}},[{visible:planningCapture.screen,pending:call},{visible:planningCapture.screen,pending:call}],{});
const owned=config===planningCapture.pendingRecord.configDir;
expect(calls).toBe(owned?1:0);expect(result.sent).toEqual(owned?['1']:[]);
expect(result.seen.length>0).toBe(owned);
});
test('opt-in unbound multi-tab redraw waits without calling picker or consuming seen state, then answers matched render',async()=>{
const first=shortQuestion('First'),second=shortQuestion('Second');const call=pending(0,first);call.questions.push(second);
let calls=0;
const result=await drive({requireNativePicker:true,pickAUQ:()=>{calls++;return 2;}},[
{visible:frame(first),pending:call}, {visible:frame(first,call.questions),pending:call},
],{cwd:'unused',deadlineAt:Date.now()+10000});
expect(result.checkpoints).toEqual([{seen:0,sent:0},{seen:0,sent:0}]);expect(calls).toBe(1);expect(result.sent).toEqual(['2']);
expect(result.seen).toContain(`${call.sessionId}:${call.toolUseId}:question:0`);
});
test('absent native identity and stale single-question render cannot invoke the required actor',async()=>{
const q=shortQuestion('Current'),other=shortQuestion('Foreign'),call=pending(0,q);let calls=0;
const result=await drive({requireNativePicker:true,pickAUQ:()=>{calls++;return 1;}},[
{visible:frame(q)}, {visible:frame(other),pending:call},
],{});
expect(calls).toBe(0);expect(result.sent).toEqual([]);expect(result.seen).toEqual([]);
});
for(const value of [null,undefined])test('bound required picker cannot fall through on '+String(value),async()=>{
const q=shortQuestion('Current'),call=pending(0,q);let calls=0;
await expect(drive({requireNativePicker:true,pickAUQ:()=>{calls++;return value;}},[{visible:frame(q),pending:call}],{})).rejects.toThrow('no authorized choice');
expect(calls).toBe(1);
});
test('the default multi-tab fallback remains unchanged when opt-in is absent',async()=>{
const q=shortQuestion('First'),call=pending(0,q);call.questions.push(shortQuestion('Second'));let calls=0;
const result=await drive({pickAUQ:()=>{calls++;return 2;}},[{visible:frame(q),pending:call}],{});
expect(calls).toBe(0);expect(result.sent).toEqual(['1']);expect(result.seen.length).toBeGreaterThan(0);
});
test('declared optional prerequisite decline precedes catalog authority only on its matched native tab',async()=>{
const q:NativeQuestion={header:'Office hours',question:'There is no design doc. Run /office-hours or proceed?',multiSelect:false,
options:[{label:'Run /office-hours first',description:'Produce a design doc.'},{label:'Skip — standard review',description:'Proceed with the supplied plan.'}]};
const call=pending(0,q);let calls=0;
const result=await drive({requireNativePicker:true,pickAUQ:()=>{calls++;throw Error('not the catalog');}},[{visible:frame(q),pending:call}],{});
expect(calls).toBe(0);expect(result.sent).toEqual(['2']);
});
test('opt-in requires a picker before creating any native fixture',async()=>{
await expect(runPlanSkillCounting({requireNativePicker:true} as any)).rejects.toThrow('requires a declared picker');
});
for(const scenario of ['controlled','original-d11','modified-choice'] as const)test('actual registration declares the finite actor and unchanged terminal budget: '+scenario,async()=>{
const temp=fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(),'eng-catalog-registration-')));
const script=path.join(temp,'registered.test.ts'),factsPath=path.join(temp,'facts.json');
const helper=(name:string)=>path.join(ROOT,'test/helpers',name);
fs.writeFileSync(script,`
import {describe,expect,mock} from 'bun:test';import * as fs from 'node:fs';import * as path from 'node:path';
import captured from ${JSON.stringify(path.join(ROOT,'test/fixtures/eng-count-actor-491.json'))};
import oldTerminal from ${JSON.stringify(path.join(ROOT,'test/fixtures/eng-fb10-count-public.json'))};
import {ENG_COUNT_COMMITMENTS} from ${JSON.stringify(helper('eng-count-question-policy.ts'))};
import * as imported from ${JSON.stringify(helper('claude-pty-runner.ts'))};const actual={...imported};
const facts={actors:0,judges:0,answers:[],required:false,originalSeed:false,report:''};
const save=()=>fs.writeFileSync(${JSON.stringify(factsPath)},JSON.stringify(facts));
const source=fs.readFileSync(${JSON.stringify(path.join(ROOT,'test/skill-e2e-plan-eng-finding-count.test.ts'))},'utf8');
const terminator=${JSON.stringify("].join('\\n');")};
const a=source.indexOf('const planEng5Findings = '),b=source.indexOf(terminator,a);
if(a<0||b<=a)throw Error('Missing actual original-seed builder');
const seed=new Function(new Bun.Transpiler({loader:'ts'}).transformSync(source.slice(a,b+terminator.length))+';return planEng5Findings;')();
mock.module(${JSON.stringify(helper('e2e-gate.ts'))},()=>({describeE2ETier:t=>{expect(t).toBe('periodic');return describe;}}));
mock.module(${JSON.stringify(helper('eng-seeded-coverage.ts'))},()=>({evaluateEngTerminalReview:async(plan,input)=>{
facts.judges++;facts.originalSeed=plan===seed(facts.report);save();expect(facts.originalSeed).toBe(true);
expect(plan).not.toContain('Declared review actor interface');expect(input.deadlineAt).toBeLessThanOrEqual(Date.now()+1_500_000);
return {administrativeCallIds:[],substantiveCallIds:[]};
}}));
mock.module(${JSON.stringify(helper('claude-pty-runner.ts'))},()=>({...actual,runPlanSkillCounting:async opts=>{
facts.actors++;facts.required=opts.requireNativePicker;facts.report=opts.expectedPlanPath;save();
expect(opts.requireNativePicker).toBe(true);expect(opts.observeSetupQuestions).toBe(true);expect(opts.preconfiguredReviewActor).toBe(true);
expect(opts.defaultPick).toBeUndefined();expect(opts.model).toBeUndefined();expect(opts.reviewCountCeiling).toBe(Infinity);
expect(opts.timeoutMs).toBeLessThanOrEqual(1_500_000);expect(opts.timeoutMs).toBeGreaterThan(1_499_000);
expect(opts.env).toEqual({QUESTION_TUNING:'false',EXPLAIN_LEVEL:'default'});
const original=seed(opts.expectedPlanPath);expect(opts.followUpPrompt.startsWith(original+'\\n\\n')).toBe(true);
fs.writeFileSync(${JSON.stringify(path.join(temp,'PLAN.md'))},opts.followUpPrompt);
for(let i=0;i<13;i++){
const call=structuredClone(captured.calls[i]);call.answered=false;call.failed=false;delete call.answers;delete call.answeredAt;
const q=call.questions[0];
if(${JSON.stringify(scenario)}!=='original-d11'||i!==10){const r=ENG_COUNT_COMMITMENTS.find(row=>row.id===captured.controlledReplacements.selectedIds[i]);q.options[captured.controlledReplacements.replaceIndices[i]]={label:r.label,description:r.description};}
if(${JSON.stringify(scenario)}==='modified-choice'&&i===0)q.options[0].description+=' Contact another provider.';
const screen='☐ '+q.header+'\\n'+q.question+'\\n'+q.options.map((o,j)=>(j===0?'❯ ':' ')+(j+1)+'. '+o.label+'\\n '+o.description).join('\\n')+'\\n '+(q.options.length+1)+'. Type something.\\n '+(q.options.length+2)+'. Chat about this\\nEnter to select · ↑/↓ to navigate · Esc to cancel';
const fp=actual.capturePlanCountQuestion(screen,new Set(),1,false,call);expect(fp?.nativeCall).toBe(call);
const picked=opts.pickAUQ(fp,fp,{cwd:${JSON.stringify(temp)},deadlineAt:Date.now()+10000});facts.answers.push({i,picked,label:q.options[picked-1].label});save();
}
const report=oldTerminal.report.replace('| Review | Skill | Runs | Status | Last run | Notes |','| Review | Trigger | Runs | Status | Why | Findings |');
fs.writeFileSync(opts.expectedPlanPath,report);
await opts.evaluateTerminal({transcript:{status:'ready',calls:[],assistantMessages:[]},report,reportMtimeMs:Date.now(),startedAt:Date.now(),finishedAt:Date.now(),deadlineAt:Date.now()+1_500_000});
return {outcome:'plan_ready',elapsedMs:1,step0Count:0,reviewCount:4,fingerprints:[],transcript:{status:'ready',calls:[],assistantMessages:[]},evidence:'controlled callback transport, no behavioral credit'};
}}));
await import(${JSON.stringify(path.join(ROOT,'test/skill-e2e-plan-eng-finding-count.test.ts'))});
`);
try{
const child=Bun.spawn([process.execPath,'test',script],{cwd:ROOT,stdout:'pipe',stderr:'pipe',timeout:10000,
env:{PATH:process.env.PATH??'',HOME:temp,TMPDIR:temp,TEMP:temp,TMP:temp,EVALS_HERMETIC:'1',GIT_CONFIG_NOSYSTEM:'1'}});
const [out,err,code]=await Promise.all([new Response(child.stdout).text(),new Response(child.stderr).text(),child.exited]);
expect(fs.existsSync(factsPath),out+'\n'+err).toBe(true);
const facts=JSON.parse(fs.readFileSync(factsPath,'utf8'));
expect(code,out+'\n'+err).toBe(scenario==='controlled'?0:1);expect(facts.actors).toBe(1);expect(facts.required).toBe(true);
expect(facts.answers).toHaveLength(scenario==='controlled'?13:scenario==='original-d11'?10:0);
expect(facts.judges).toBe(scenario==='controlled'?1:0);
if(scenario==='controlled'){expect(facts.originalSeed).toBe(true);expect(facts.answers[10].label).toBe('Keep current behavior');}
else expect(err).toContain('exactly one complete author-owned offered commitment');
expect(fs.existsSync(facts.report)).toBe(false);
}finally{fs.rmSync(temp,{recursive:true,force:true});}
});