diff --git a/test/autoplan-setup-packet-o.test.ts b/test/autoplan-setup-packet-o.test.ts index 7517b797f..efa996111 100644 --- a/test/autoplan-setup-packet-o.test.ts +++ b/test/autoplan-setup-packet-o.test.ts @@ -9,6 +9,7 @@ import type { NativePlanQuestionCall } from './helpers/plan-count-transcript'; const captured = fs.readFileSync(path.join(import.meta.dir, 'fixtures/autoplan-setup-packet-o-screen.txt'), 'utf8'); const original = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/autoplan-setup-packet-o-call.json'), 'utf8')) as NativePlanQuestionCall; +const zPacket = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/autoplan-setup-z-packet.json'), 'utf8')) as {pendingCall: NativePlanQuestionCall; screen: string}; const footer = 'Enter to select · Tab/Arrow keys to navigate · Esc to cancel'; function pane(call: NativePlanQuestionCall, index: number, answered: number[] = []) { const bar = '← ' + call.questions.map((q,i) => (answered.includes(i) ? '☒ ' : '☐ ') + q.header).join(' ') + ' ✔ Submit →'; @@ -117,11 +118,83 @@ describe('native routing and prerequisite setup packet', () => { }); }); +describe('numbered native setup packet preserves the full existing review', () => { + test('exact Z packet advances both bound tabs and only then submits once', () => { + const seen = new Set(); + expect(autoplanSetupDecision(zPacket.screen, seen).kind).toBe('waiting'); + commit(zPacket.screen, seen, zPacket.pendingCall, '1'); + expect(autoplanSetupDecision(pane(zPacket.pendingCall, 2, [0,1]), seen, zPacket.pendingCall).kind).toBe('waiting'); + commit(pane(zPacket.pendingCall, 1, [0]), seen, zPacket.pendingCall, '1'); + commit(pane(zPacket.pendingCall, 2, [0,1]), seen, zPacket.pendingCall, '\r'); + expect(E2E_TOUCHFILES['autoplan-chain-pty']).toContain('test/fixtures/autoplan-setup-z-packet.json'); + }); + + test('numbering and offered order vary while picks keep their exact native identities', () => { + for (const reverseQuestions of [false, true]) for (const reverseOptions of [false, true]) { + const call = structuredClone(zPacket.pendingCall); + call.questions[0]!.question = call.questions[0]!.question.replace('D1 —', 'D17:'); + call.questions[1]!.question = call.questions[1]!.question.replace('D2 —', 'D23 –').replace('this branch', 'the project').replace('the review input', 'this review input'); + if (reverseQuestions) call.questions.reverse(); + if (reverseOptions) call.questions.forEach(question => question.options.reverse()); + const seen = new Set(); + commit(pane(call,0), seen, call, reverseOptions ? '2' : '1'); + commit(pane(call,1,[0]), seen, call, reverseOptions ? '2' : '1'); + commit(pane(call,2,[0,1]), seen, call, '\r'); + } + }); + + test('all new question and option description clauses must remain setup only', () => { + const mutations: Array<(call: NativePlanQuestionCall) => void> = [ + call => { call.questions[0]!.question += ' Also remove account-owner authorization.'; }, + call => { call.questions[1]!.question += ' Approve dropping the audit tests?'; }, + call => { call.questions[0]!.question = 'The plan quotes ' + call.questions[0]!.question; }, + call => { call.questions[1]!.question = call.questions[1]!.question.replace('sharpen the review input', 'approve the proposed changes'); }, + call => { call.questions[1]!.options[0]!.description = call.questions[1]!.options[0]!.description!.replace('CEO → Design → DX → Eng', 'CEO → Eng'); }, + call => { call.questions[1]!.options[0]!.description = call.questions[1]!.options[0]!.description!.replace('plan as-is', 'plan after removing authorization'); }, + call => { call.questions[1]!.options[0]!.label += ' and implement'; }, + call => { call.questions[1]!.options[1]!.label += ' then ship'; }, + ]; + for (let question = 0; question < 2; question++) for (let option = 0; option < 2; option++) { + mutations.push(call => { call.questions[question]!.options[option]!.description += ' Also delete the account-owner check.'; }); + mutations.push(call => { call.questions[question]!.options[option]!.description = undefined; }); + } + for (const mutate of mutations) { + const call = structuredClone(zPacket.pendingCall); mutate(call); + const seen = new Set(); + expect(autoplanSetupDecision(pane(call,0), seen, call).kind).toBe('waiting'); + expect(seen.size).toBe(0); + } + }); + + test('new forms require complete pending native identity and the same intact active pane', () => { + const mutations: Array<(call: any) => void> = [ + call => { delete call.answered; }, call => { delete call.failed; }, call => { call.answered = true; }, call => { call.failed = true; }, + call => { delete call.sessionId; }, call => { delete call.toolUseId; }, + call => { call.questions[0].question = call.questions[0].question.replace('routing-injection', 'other-question'); }, + call => { call.questions[0].question += ' '; }, + call => { call.questions[1].question = call.questions[1].question.replace('D2', 'D0'); }, + call => { call.questions[1].multiSelect = true; }, + call => { call.questions.push(structuredClone(call.questions[0])); }, + call => { call.questions[1] = structuredClone(call.questions[0]); }, + ]; + for (const mutate of mutations) { const call = structuredClone(zPacket.pendingCall); mutate(call); + expect(autoplanSetupDecision(pane(call,0),new Set(),call).kind).toBe('waiting'); } + const first = pane(zPacket.pendingCall,0); + for (const screen of ['Example panel:\n'+first, '```text\n'+first, first+'\nProceeding.', first.replace('Esc to cancel','Esc to'), + first.replace('Design doc','Other tab'), first.replace('1. Add','1. Delete'), first.replace('← ','← Unrelated packet '), + first.split('\n').map(line => '> '+line).join('\n')]) { + expect(autoplanSetupDecision(screen,new Set(),zPacket.pendingCall).kind).toBe('waiting'); + } + expect(autoplanSetupDecision(pane(zPacket.pendingCall,2,[0,1]),new Set(),zPacket.pendingCall).kind).toBe('waiting'); + }); +}); + test.skipIf(process.platform==='win32')('real PTY native setup packet waits for metadata, answers each visible tab once and submits without a stray digit',async()=>{ const dir=fs.mkdtempSync(path.join(os.tmpdir(),'gstack-setup-packet-o-'));const fake=path.join(dir,'fake-claude'); const worker=path.join(dir,'worker.ts');const resultFile=path.join(dir,'results.json'); - const cases=[false,true].map(late=>({name:late?'late':'early',late,cwd:path.join(dir,late?'late':'early'), - events:path.join(dir,late?'late.jsonl':'early.jsonl'),release:path.join(dir,late?'late.release':'early.release'),call:original,first:captured})); + const cases=[{name:'o',call:original,first:captured},{name:'z',call:zPacket.pendingCall,first:zPacket.screen}].flatMap(packet => + [false,true].map(late=>({name:packet.name+(late?'-late':'-early'),late,cwd:path.join(dir,packet.name+(late?'-late':'-early')), + events:path.join(dir,packet.name+(late?'-late.jsonl':'-early.jsonl')),release:path.join(dir,packet.name+(late?'-late.release':'-early.release')),call:packet.call,first:packet.first}))); for(const item of cases)fs.mkdirSync(item.cwd); fs.writeFileSync(fake,`#!${process.execPath}\n`+String.raw` import * as fs from 'node:fs';import * as path from 'node:path'; @@ -165,7 +238,7 @@ for(let step=0;step<3;step++){ if(action.kind!=='input')throw Error('Expected input at '+step+': '+JSON.stringify({action,current,call})); session.send(action.input);inputs.push(action.input);for(const signature of action.signatures)seen.add(signature); if(autoplanSetupDecision(current,seen,call).kind!=='waiting')throw Error('Repeated input on unchanged pane'); - await session.waitFor(step===0?'☒ Routing setup':step===1?'Ready to submit your answers?':'NATIVE_PACKET_COMPLETE',{timeoutMs:3000,pollMs:20}); + await session.waitFor(step===0?'☒ '+item.call.questions[0].header:step===1?'Ready to submit your answers?':'NATIVE_PACKET_COMPLETE',{timeoutMs:3000,pollMs:20}); } const transcript=readPlanCountTranscript(session.hermeticConfigDir,item.cwd);results.push({name:item.name,inputs,transcript}); }finally{await session.close();}} @@ -175,10 +248,10 @@ fs.writeFileSync(${JSON.stringify(resultFile)},JSON.stringify(results)); const killer=setTimeout(()=>child.kill('SIGKILL'),26000); try{ const [exit,stdout,stderr]=await Promise.all([child.exited,new Response(child.stdout).text(),new Response(child.stderr).text()]);expect(exit,stdout+stderr).toBe(0); - const results=JSON.parse(fs.readFileSync(resultFile,'utf8'));expect(results.length).toBe(2); + const results=JSON.parse(fs.readFileSync(resultFile,'utf8'));expect(results.length).toBe(4); for(const [index,result]of results.entries()){ expect(result.inputs).toEqual(['1','1','\r']);expect(result.transcript.calls.length).toBe(1);expect(result.transcript.calls[0].answered).toBe(true); - expect(result.transcript.calls[0].answers).toEqual(Object.fromEntries(original.questions.map(q=>[q.question,q.options[0]!.label]))); + expect(result.transcript.calls[0].answers).toEqual(Object.fromEntries(cases[index]!.call.questions.map(q=>[q.question,q.options[0]!.label]))); const events=fs.readFileSync(cases[index]!.events,'utf8').trim().split('\n').map(line=>JSON.parse(line)); expect(events.filter(e=>e.kind==='input').map(e=>({input:e.input,index:e.index,published:e.published}))).toEqual([ {input:'1',index:0,published:true},{input:'1',index:1,published:true},{input:'\r',index:2,published:true}]); diff --git a/test/ceo-handoff-y.test.ts b/test/ceo-handoff-y.test.ts index 1699c9685..ebbfa29bb 100644 --- a/test/ceo-handoff-y.test.ts +++ b/test/ceo-handoff-y.test.ts @@ -3,6 +3,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import fixture from './fixtures/ceo-handoff-y-call.json'; +import zFixture from './fixtures/ceo-handoff-z-call.json'; import type {NativePlanQuestionCall} from './helpers/plan-count-transcript'; import {ceoFirstReviewAUQ,ceoStep0Boundary,hasNativePlanTerminal,nativePlanCallFingerprint,planCountQuestionPhase} from './helpers/claude-pty-runner'; import {isCeoCompletionHandoff,pickCeoCompletionHandoff} from './helpers/ceo-completion-handoff'; @@ -48,3 +49,43 @@ describe('Y bare next-Eng navigation is administrative, not completion evidence' }finally{fs.rmSync(dir,{recursive:true,force:true});} }); }); + +describe('Z completed CEO with an unrun required Eng gate',()=>{ + const actualZ=()=>structuredClone(zFixture.calls.at(-1)!) as NativePlanQuestionCall; + const pendingZ=(c=actualZ())=>{c.answered=false;delete c.answers;delete c.answeredAt;delete c.unansweredQuestionIndices;return c;}; + const reject=(c:NativePlanQuestionCall)=>{expect(isCeoCompletionHandoff(fp(c))).toBe(false);expect(pickCeoCompletionHandoff(fp(c))).toBeNull();}; + test('exact six calls preserve two findings; handoff selects the offered manual route',()=>{ + let started=false;const counts={setup:0,review:0,admin:0}; + for(const c of zFixture.calls){const phase=planCountQuestionPhase(fp(c as NativePlanQuestionCall),started,ceoStep0Boundary,ceoFirstReviewAUQ,undefined,isCeoCompletionHandoff);started=phase.reviewStarted;counts[phase.administrative?'admin':phase.preReview?'setup':'review']++;} + expect(counts).toEqual({setup:3,review:2,admin:1});expect(isCeoCompletionHandoff(fp(actualZ()))).toBe(true);expect(pickCeoCompletionHandoff(fp(pendingZ()))).toBe(2); + expect(planCountQuestionPhase(fp(actualZ()),false,ceoStep0Boundary,ceoFirstReviewAUQ,undefined,isCeoCompletionHandoff)).toEqual({preReview:false,reviewStarted:false,administrative:'completion-handoff'}); + }); + test('number, typography and option order are not semantic requirements',()=>{ + const c=change(actualZ(),s=>s.replace('D5 —','D27:').replace("hasn't",'has not').replace("What's",'What is'));c.questions[0]!.options.reverse(); + for(const option of c.questions[0]!.options){c.answers={[c.questions[0]!.question]:option.label};expect(isCeoCompletionHandoff(fp(c))).toBe(true);} + expect(pickCeoCompletionHandoff(fp(pendingZ(c)))).toBe(1); + }); + test('whole question and role-specific descriptions cannot hide new or conditional work',()=>{ + for(const fn of [(s:string)=>s.replace('CEO Review is CLEAR','If CEO Review is CLEAR'),(s:string)=>s.replace('CEO Review is CLEAR','CEO Review is not CLEAR'),(s:string)=>s.replace('required shipping gate','optional shipping gate'),(s:string)=>s.replace("What's next?","What's next? Also add retries."),(s:string)=>'> '+s,(s:string)=>'Example: '+s,(s:string)=>'```\n'+s+'\n```',(s:string)=>s.replace('plan-ceo-next-review','foreign-next-review'),(s:string)=>s+' '])reject(change(actualZ(),fn)); + for(const i of [0,1])for(const extra of [' Also implement the missing checks.',' Rotate credentials.',' Should we add a new requirement?',' Once remaining findings are fixed.']){const c=actualZ();c.questions[0]!.options[i]!.description+=extra;reject(c);} + const swapped=actualZ();[swapped.questions[0]!.options[0]!.description,swapped.questions[0]!.options[1]!.description]=[swapped.questions[0]!.options[1]!.description,swapped.questions[0]!.options[0]!.description];reject(swapped); + const optional=actualZ();optional.questions[0]!.options[1]!.description=optional.questions[0]!.options[1]!.description!.replace('required before shipping','optional before shipping');reject(optional); + }); + test('new arm requires explicit native completion and exact producer pending state',()=>{ + const mutations=[(c:NativePlanQuestionCall)=>{delete c.failed;},(c:NativePlanQuestionCall)=>{c.failed=true;},(c:NativePlanQuestionCall)=>{c.sessionId='';},(c:NativePlanQuestionCall)=>{c.toolUseId='';},(c:NativePlanQuestionCall)=>{c.questions[0]!.header='Issue';},(c:NativePlanQuestionCall)=>{c.questions[0]!.multiSelect=true;},(c:NativePlanQuestionCall)=>{c.questions.push(structuredClone(c.questions[0]!));},(c:NativePlanQuestionCall)=>{c.questions[0]!.options.push({label:'Add a repair',description:'Add a new requirement.'});}]; + for(const mutate of mutations){const c=actualZ();mutate(c);reject(c);const p=pendingZ();mutate(p);reject(p);} + for(const mutate of [(c:NativePlanQuestionCall)=>{delete c.unansweredQuestionIndices;},(c:NativePlanQuestionCall)=>{c.unansweredQuestionIndices=[0];},(c:NativePlanQuestionCall)=>{c.answers={[c.questions[0]!.question]:'Repair first'};}]){const c=actualZ();mutate(c);reject(c);} + for(const mutate of [(c:NativePlanQuestionCall)=>{delete (c as any).answered;},(c:NativePlanQuestionCall)=>{c.unansweredQuestionIndices=[];},(c:NativePlanQuestionCall)=>{c.unansweredQuestionIndices=[1];},(c:NativePlanQuestionCall)=>{c.answers={};},(c:NativePlanQuestionCall)=>{c.answeredAt='2026-09-09T12:00:00Z';}]){const c=pendingZ();mutate(c);reject(c);} + const projected=pendingZ();projected.unansweredQuestionIndices=[0];expect(pickCeoCompletionHandoff(fp(projected))).toBe(2); + for(const variant of [{...fp(pendingZ()),signature:'foreign:call'},{...fp(pendingZ()),options:[]},{...fp(pendingZ()),nativeQuestionIndex:1}])expect(pickCeoCompletionHandoff(variant)).toBeNull(); + }); + test('retained original mtime passes only with the administrative handoff and real Exit',()=>{ + const dir=fs.mkdtempSync(path.join(os.tmpdir(),'gstack-handoff-z-free-'));const report=path.join(dir,'report.md'); + try{fs.writeFileSync(report,zFixture.report);const calls=structuredClone(zFixture.calls) as NativePlanQuestionCall[];const transcript={status:'ready' as const,calls,assistantMessages:[],planReadyRequests:structuredClone(zFixture.planReadyRequests)};const admin=new Set(calls.filter(c=>isCeoCompletionHandoff(fp(c))).map(c=>fp(c).signature));const mtime=Number(BigInt(zFixture.reportOriginalMtimeNs))/1e6;fs.utimesSync(report,mtime/1000,mtime/1000); + expect(hasNativePlanTerminal(transcript,report,zFixture.startedAt,'plan_ready')).toBe(false);expect(hasNativePlanTerminal(transcript,report,zFixture.startedAt,'plan_ready',admin)).toBe(true); + expect(hasNativePlanTerminal({...transcript,planReadyRequests:[]},report,zFixture.startedAt,'plan_ready',admin)).toBe(false); + expect(hasNativePlanTerminal({...transcript,calls:[calls.at(-1)!]},report,zFixture.startedAt,'plan_ready',admin)).toBe(false); + const stale=Date.parse(calls.at(-2)!.answeredAt!)-1;fs.utimesSync(report,stale/1000,stale/1000);expect(hasNativePlanTerminal(transcript,report,zFixture.startedAt,'plan_ready',admin)).toBe(false); + }finally{fs.rmSync(dir,{recursive:true,force:true});} + }); +}); diff --git a/test/design-count-review.test.ts b/test/design-count-review.test.ts index 2ce7aba61..15dd1c26e 100644 --- a/test/design-count-review.test.ts +++ b/test/design-count-review.test.ts @@ -13,6 +13,7 @@ import numberedPasses from './fixtures/design-review-l-calls.json'; import scoredPasses from './fixtures/design-review-n-calls.json'; import outsideCalls from './fixtures/design-outside-y-calls.json'; import boundaryCalls from './fixtures/design-boundaries-y-calls.json'; +import gapCalls from './fixtures/design-gap-z-calls.json'; const calls = () => structuredClone(captured.calls) as NativePlanQuestionCall[]; const fingerprint = (call: NativePlanQuestionCall) => nativePlanCallFingerprint(call, 0, true); @@ -40,6 +41,35 @@ function replay(input: NativePlanQuestionCall[], first = isDesignCountFirstRevie return { ...counts, started, phases }; } +describe('Z numbered gap starts review at the actual plan amendment', () => { + const actual = () => structuredClone(gapCalls.calls) as NativePlanQuestionCall[]; + const first = () => actual()[0]!; + const reanswer = (c: NativePlanQuestionCall) => { c.answers = {[c.questions[0]!.question]:c.questions[0]!.options[0]!.label}; return c; }; + test('first visual hierarchy decision opens all eight substantive calls without changing raw count', () => { + expect(isDesignCountFirstReview(fingerprint(first()))).toBe(true); + const output = replay(actual()); + expect(output).toMatchObject({step0:0,review:8,administrative:0,started:true}); + expect(output.phases).toHaveLength(8);expect(output.phases.every(p=>!p.preReview&&!p.administrative)).toBe(true); + expect(isDesignCompletionHandoff(fingerprint(first()))).toBe(false); + expect(isDesignCountSetup(fingerprint(first()))).toBe(false); + }); + test('control names, palette, gap/task numbers and offered answer order may vary', () => { + const c=first(),q=c.questions[0]!; + q.question=q.question.replace('Gap 1 of 8','Gap 3 of 12').replace('Save button','Submit button');q.header='Gap 3: Button'; + q.options[0]!.description=q.options[0]!.description!.replace('Save gets #1d4ed8','Submit gets #123abc').replace('T1','T9');q.options.reverse(); + for(const o of q.options){c.answers={[q.question]:o.label};expect(isDesignCountFirstReview(fingerprint(c))).toBe(true);} + }); + test('setup, examples, mismatched finding identity and unknown offered clauses cannot open review', () => { + for(const change of [(s:string)=>s.replace('apply DESIGN.md primary button style?','start reviewing the design?'),(s:string)=>s.replace('Gap 1 of 8','Gap 9 of 8'),(s:string)=>s.replace('Gap 1','Gap 0'),(s:string)=>'Example: '+s,(s:string)=>'> '+s,(s:string)=>'```\n'+s+'\n```',(s:string)=>s+' Ready to begin?']){const c=first();c.questions[0]!.question=change(c.questions[0]!.question);expect(isDesignCountFirstReview(fingerprint(reanswer(c)))).toBe(false);} + for(const i of [0,1])for(const suffix of [' Review starts after this setup choice.',' Choose the design source first.',' Should we review the button?']){const c=first();c.questions[0]!.options[i]!.description+=suffix;expect(isDesignCountFirstReview(fingerprint(c))).toBe(false);} + for(const mutate of [(c:NativePlanQuestionCall)=>{c.questions[0]!.header='Gap 2: Button';},(c:NativePlanQuestionCall)=>{c.questions[0]!.header='Focus';},(c:NativePlanQuestionCall)=>{c.questions[0]!.options[0]!.description=c.questions[0]!.options[0]!.description!.replace('Save gets','Publish gets');},(c:NativePlanQuestionCall)=>{c.questions[0]!.options[0]!.label='Start the review';},(c:NativePlanQuestionCall)=>{c.questions[0]!.options[1]!.label='Wait';}]){const c=first();mutate(c);expect(isDesignCountFirstReview(fingerprint(reanswer(c)))).toBe(false);} + }); + test('only one explicitly completed current native question with an offered answer opens review', () => { + for(const mutate of [(c:NativePlanQuestionCall)=>{c.answered=false;},(c:NativePlanQuestionCall)=>{delete (c as Partial).answered;},(c:NativePlanQuestionCall)=>{c.failed=true;},(c:NativePlanQuestionCall)=>{delete c.failed;},(c:NativePlanQuestionCall)=>{c.sessionId='';},(c:NativePlanQuestionCall)=>{c.toolUseId='';},(c:NativePlanQuestionCall)=>{delete c.unansweredQuestionIndices;},(c:NativePlanQuestionCall)=>{c.unansweredQuestionIndices=[0];},(c:NativePlanQuestionCall)=>{c.answers={};},(c:NativePlanQuestionCall)=>{c.answers={[c.questions[0]!.question]:'Start reviewing'};},(c:NativePlanQuestionCall)=>{c.questions[0]!.multiSelect=true;},(c:NativePlanQuestionCall)=>{c.questions.push(structuredClone(c.questions[0]!));},(c:NativePlanQuestionCall)=>{c.questions[0]!.options.push({label:'Another choice'});}]){const c=first();mutate(c);expect(isDesignCountFirstReview(fingerprint(c))).toBe(false);} + for(const f of [{...fingerprint(first()),signature:'foreign:call'},{...fingerprint(first()),nativeCall:undefined},{...fingerprint(first()),nativeQuestionIndex:1},{...fingerprint(first()),options:[]}])expect(isDesignCountFirstReview(f)).toBe(false); + }); +}); + describe('Native finding and closed handoff boundaries', () => { const actual = () => structuredClone(boundaryCalls) as NativePlanQuestionCall[]; const handoff = () => actual().at(-1)!; diff --git a/test/devex-count-fixture.test.ts b/test/devex-count-fixture.test.ts index e566909e8..a76bc2c61 100644 --- a/test/devex-count-fixture.test.ts +++ b/test/devex-count-fixture.test.ts @@ -1,3 +1,4 @@ +import capturedZ from './fixtures/devex-count-z-calls.json'; import capturedURetry from './fixtures/devex-count-u-retry-calls.json'; import capturedY from './fixtures/devex-count-y-calls.json'; import capturedV from './fixtures/devex-empathy-v-calls.json'; @@ -630,3 +631,50 @@ describe('U demo timing decision after completed measurements', () => { expect(isDevexReviewIssue({...fp(captured()), nativeCall:undefined})).toBe(false); }); }); + +describe('Z written migration guide as an additional accepted obligation', () => { + const call = () => structuredClone(capturedZ[7]!) as NativePlanQuestionCall; + const fp = (c: NativePlanQuestionCall) => nativePlanCallFingerprint(c, 0, false); + const change = (c: NativePlanQuestionCall, from: string, to: string) => { + const q=c.questions[0]!;const answer=c.answers![q.question]!;q.question=q.question.replaceAll(from,to);c.answers={[q.question]:answer};return c; + }; + test('all eight real calls retain empathy plus seven distinct issue decisions', () => { + const calls=structuredClone(capturedZ) as NativePlanQuestionCall[]; + expect(calls.map(c=>isDevexReviewIssue(fp(c)))).toEqual([false,true,true,true,true,true,true,true]);expect(calls).toEqual(capturedZ); + const c=call();c.questions[0]!.options.reverse();expect(isDevexReviewIssue(fp(c))).toBe(true); + }); + test('version, decision and task numbers do not determine finding credit', () => { + const c=call();for(const [from,to] of [['D8','D17'],['TODO-2','TODO-9'],['todo2-migration','todo9-migration'],['v1','v3'],['v2','v4'],['T4','T11'],['P2','P1']]) { + change(c,from!,to!);const q=c.questions[0]!;q.header=q.header.replaceAll(from!,to!);q.options.forEach(o=>{o.description=o.description?.replaceAll(from!,to!);}); + }expect(isDevexReviewIssue(fp(c))).toBe(true); + }); + test('setup, quoted, hypothetical and already satisfied claims confer no new acceptance', () => { + for(const [from,to] of [ + ['TODO: should the plan include','TODO: should the review confirm'], + ['But there is currently no written migration guide in docs/.','The written migration guide already exists in docs/.'], + ['But there is currently no written migration guide in docs/.','But there is currently no written migration guide in docs/ only in this hypothetical example.'], + ['The deprecation shim (T4) handles','If the deprecation shim (T4) handles'], + ['The deprecation shim (T4) handles','> The deprecation shim (T4) handles'], + ['The deprecation shim (T4) handles','```text\nThe deprecation shim (T4) handles'], + ['A one-page migration guide covers:','The already-approved migration guide covers:'], + ['Without it, developers','This is only an example. Without it, developers'], + ['',''], + ['',''], + ])expect(isDevexReviewIssue(fp(change(call(),from!,to!)))).toBe(false); + for(const header of ['Review mode','Empathy check','Next steps']){const c=call();c.questions[0]!.header=header;expect(isDevexReviewIssue(fp(c))).toBe(false);} + expect(isDevexReviewIssue(fp(change(call(),'',' ')))).toBe(false); + }); + test('one exact successful native call must select the offered written-guide task', () => { + for(const mutate of [ + (c:NativePlanQuestionCall)=>{c.answered=false;},(c:NativePlanQuestionCall)=>{c.failed=true;},(c:NativePlanQuestionCall)=>{delete c.failed;}, + (c:NativePlanQuestionCall)=>{delete c.unansweredQuestionIndices;},(c:NativePlanQuestionCall)=>{c.unansweredQuestionIndices=[0];}, + (c:NativePlanQuestionCall)=>{c.answers={};},(c:NativePlanQuestionCall)=>{c.answers={[c.questions[0]!.question]:'unoffered answer'};}, + (c:NativePlanQuestionCall)=>{c.questions[0]!.multiSelect=true;},(c:NativePlanQuestionCall)=>{c.questions.push(structuredClone(c.questions[0]!));}, + (c:NativePlanQuestionCall)=>{c.questions[0]!.options.push(structuredClone(c.questions[0]!.options[0]!));}, + (c:NativePlanQuestionCall)=>{c.questions[0]!.options[0]!.description+=' Also remove authentication.';}, + (c:NativePlanQuestionCall)=>{c.questions[0]!.options[0]!.description=undefined;}, + ]){const c=call();mutate(c);expect(isDevexReviewIssue(fp(c))).toBe(false);} + for(const index of [1,2]){const c=call();const q=c.questions[0]!;c.answers={[q.question]:q.options[index]!.label};expect(isDevexReviewIssue(fp(c))).toBe(false);} + expect(isDevexReviewIssue({...fp(call()),signature:'foreign:call'})).toBe(false);expect(isDevexReviewIssue({...fp(call()),nativeCall:undefined})).toBe(false); + }); +}); diff --git a/test/eng-binding-retry-z.test.ts b/test/eng-binding-retry-z.test.ts new file mode 100644 index 000000000..dce4baa74 --- /dev/null +++ b/test/eng-binding-retry-z.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, test } from 'bun:test'; +import captured from './fixtures/eng-binding-retry-z-calls.json'; +import { engFirstReviewAUQ, engSetupAUQ, engStep0Boundary, nativePlanCallFingerprint, planCountQuestionPhase } from './helpers/claude-pty-runner'; +import type { NativePlanQuestionCall } from './helpers/plan-count-transcript'; + +const fresh = () => structuredClone(captured[1]!) as NativePlanQuestionCall; +const fp = (c: NativePlanQuestionCall) => nativePlanCallFingerprint(c, 0, true); +const first = (c: NativePlanQuestionCall) => engFirstReviewAUQ(fp(c)); +function question(c: NativePlanQuestionCall, transform: (s: string) => string) { + const q = c.questions[0]!; const answer = c.answers![q.question]!; + q.question = transform(q.question); c.answers = {[q.question]: answer}; return c; +} + +describe('Z Eng shared mutable cache starts substantive review', () => { + test('the actual shared mutable cache risk starts review without an issue label', () => { + expect(engSetupAUQ(fp(fresh()))).toBe(false); + expect(first(fresh())).toBe(true); + expect(planCountQuestionPhase(fp(fresh()), false, engStep0Boundary, engFirstReviewAUQ, engSetupAUQ)) + .toEqual({preReview: false, reviewStarted: true}); + }); + + test('the exact six native calls preserve one setup and all five review obligations', () => { + let started = false; + const phases = captured.map(c => { + const p = planCountQuestionPhase(fp(structuredClone(c) as NativePlanQuestionCall), started, engStep0Boundary, engFirstReviewAUQ, engSetupAUQ); + started = p.reviewStarted; return p.preReview; + }); + expect(phases).toEqual([true, false, false, false, false, false]); + expect(first(structuredClone(captured[0]!) as NativePlanQuestionCall)).toBe(false); + expect(captured[5]!.questions[0]!.header).toBe('TODO: E2E test'); + }); + + test('either offered choice, reordering and a different component retain issue identity', () => { + const c = fresh(); c.questions[0]!.options.reverse(); + for (const option of c.questions[0]!.options) { + c.answers = {[c.questions[0]!.question]: option.label}; expect(first(c)).toBe(true); + } + const varied = question(fresh(), s => s.replace('AuthCache', 'SessionCache').replace('D2', 'D7')); + for (const option of varied.questions[0]!.options) option.description = option.description.replaceAll('AuthCache', 'SessionCache'); + expect(first(varied)).toBe(true); + }); + + test('requires a complete native call and exact offered answer and fingerprint', () => { + for (const mutate of [ + (c: NativePlanQuestionCall) => { c.answered = false; }, + (c: NativePlanQuestionCall) => { c.failed = true; }, + (c: NativePlanQuestionCall) => { delete c.failed; }, + (c: NativePlanQuestionCall) => { delete c.unansweredQuestionIndices; }, + (c: NativePlanQuestionCall) => { c.unansweredQuestionIndices = [0]; }, + (c: NativePlanQuestionCall) => { c.questions[0]!.multiSelect = true; }, + (c: NativePlanQuestionCall) => { c.questions.push(structuredClone(c.questions[0]!)); }, + (c: NativePlanQuestionCall) => { c.questions[0]!.options.push(structuredClone(c.questions[0]!.options[0]!)); }, + (c: NativePlanQuestionCall) => { c.questions[0]!.options[1]!.label = c.questions[0]!.options[0]!.label; }, + (c: NativePlanQuestionCall) => { c.answers = {}; }, + (c: NativePlanQuestionCall) => { c.answers = {[c.questions[0]!.question]: 'Foreign answer'}; }, + ]) { const c = fresh(); mutate(c); expect(first(c)).toBe(false); } + expect(engFirstReviewAUQ({...fp(fresh()), signature: 'foreign:call'})).toBe(false); + expect(engFirstReviewAUQ({...fp(fresh()), nativeCall: undefined})).toBe(false); + expect(engFirstReviewAUQ({...fp(fresh()), options: []})).toBe(false); + const mismatch = fp(fresh()); mismatch.options[0]!.label = 'foreign'; expect(engFirstReviewAUQ(mismatch)).toBe(false); + const wrongIndex = fp(fresh()); wrongIndex.options[0]!.index = 2; expect(engFirstReviewAUQ(wrongIndex)).toBe(false); + }); + + test('requires an affirmative direct risk, not setup, denial, qualification or quotations', () => { + for (const header of ['Scope', 'Approach', 'Next review', 'Onboarding']) { + const c = fresh(); c.questions[0]!.header = header; expect(first(c)).toBe(false); + } + for (const transform of [ + (s: string) => s.replace('Architecture:', 'Approach:'), + (s: string) => s.replace('Two services share', 'If two services share'), + (s: string) => s.replace('Two services share', 'Two services do not share'), + (s: string) => s.replace('can corrupt tenant isolation', 'cannot corrupt tenant isolation'), + (s: string) => s.replace('can corrupt tenant isolation', 'never corrupt tenant isolation'), + (s: string) => s.replace('This is the #1 reliability risk', 'This is not the #1 reliability risk'), + (s: string) => s.replace('plan-eng-shared-mutable-cache', 'plan-eng-setup'), + (s: string) => s.replace('plan-eng-shared-mutable-cache', 'foreign-shared-mutable-cache'), + (s: string) => s.replace(/ ]+>/, ''), + (s: string) => s + ' ', + (s: string) => s + ' Run the next review too.', + (s: string) => '> ' + s, + (s: string) => '```text\n' + s + '\n```', + ]) expect(first(question(fresh(), transform))).toBe(false); + }); + + test('the complete offered remedies stay tied to the same dependency and affirmative risk', () => { + for (const [index, transform] of [ + [0, (s: string) => s.replace('The plan is updated', 'The plan is not updated')], + [0, (s: string) => s.replace('pass AuthCache', 'pass DifferentCache')], + [0, (s: string) => s.replace('No module-level mutable export.', 'Keep the module-level mutable export.')], + [1, (s: string) => s.replace('still couples both services', 'does not couple both services')], + [2, (s: string) => s.replace('as a known risk', 'as a dismissed risk')], + [0, (s: string) => s + ' Also grant every tenant access.'], + [1, (s: string) => s + ' Also approve the missing timeout policy.'], + [0, (s: string) => '> ' + s], + [2, (s: string) => '```text\n' + s + '\n```'], + ] as const) { + const c = fresh(); const option = c.questions[0]!.options[index]!; + option.description = transform(option.description ?? ''); expect(first(c)).toBe(false); + } + const c = fresh(); c.questions[0]!.options[0]!.label = 'Run /office-hours'; + c.answers = {[c.questions[0]!.question]: 'Run /office-hours'}; expect(first(c)).toBe(false); + }); +}); diff --git a/test/eng-binding-z.test.ts b/test/eng-binding-z.test.ts new file mode 100644 index 000000000..f6b73f67d --- /dev/null +++ b/test/eng-binding-z.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, test } from 'bun:test'; +import captured from './fixtures/eng-binding-z-calls.json'; +import { engFirstReviewAUQ, engSetupAUQ, engStep0Boundary, nativePlanCallFingerprint, planCountQuestionPhase } from './helpers/claude-pty-runner'; +import type { NativePlanQuestionCall } from './helpers/plan-count-transcript'; + +const fresh = () => structuredClone(captured[1]!) as NativePlanQuestionCall; +const fp = (c: NativePlanQuestionCall) => nativePlanCallFingerprint(c, 0, true); +const first = (c: NativePlanQuestionCall) => engFirstReviewAUQ(fp(c)); +function question(c: NativePlanQuestionCall, transform: (s: string) => string) { + const q = c.questions[0]!; const answer = c.answers![q.question]!; + q.question = transform(q.question); c.answers = {[q.question]: answer}; return c; +} + +describe('Z Eng dependency binding starts substantive review', () => { + test('the actual cache dependency decision starts review without an issue label', () => { + expect(engSetupAUQ(fp(fresh()))).toBe(false); + expect(first(fresh())).toBe(true); + expect(planCountQuestionPhase(fp(fresh()), false, engStep0Boundary, engFirstReviewAUQ, engSetupAUQ)) + .toEqual({preReview: false, reviewStarted: true}); + }); + + test('the exact six native calls preserve one setup and all five review obligations', () => { + let started = false; + const phases = captured.map(c => { + const p = planCountQuestionPhase(fp(structuredClone(c) as NativePlanQuestionCall), started, engStep0Boundary, engFirstReviewAUQ, engSetupAUQ); + started = p.reviewStarted; return p.preReview; + }); + expect(phases).toEqual([true, false, false, false, false, false]); + expect(first(structuredClone(captured[0]!) as NativePlanQuestionCall)).toBe(false); + expect(captured[5]!.questions[0]!.header).toBe('TODO: Timeout'); + }); + + test('either offered choice, reordering and a different component retain issue identity', () => { + const c = fresh(); c.questions[0]!.options.reverse(); + for (const option of c.questions[0]!.options) { + c.answers = {[c.questions[0]!.question]: option.label}; expect(first(c)).toBe(true); + } + const varied = question(fresh(), s => s.replace('AuthBroker', 'SessionGateway').replace('D2', 'D7')); + for (const option of varied.questions[0]!.options) option.description = option.description.replaceAll('AuthBroker', 'SessionGateway'); + expect(first(varied)).toBe(true); + }); + + test('requires a complete native call and exact offered answer and fingerprint', () => { + for (const mutate of [ + (c: NativePlanQuestionCall) => { c.answered = false; }, + (c: NativePlanQuestionCall) => { c.failed = true; }, + (c: NativePlanQuestionCall) => { delete c.failed; }, + (c: NativePlanQuestionCall) => { delete c.unansweredQuestionIndices; }, + (c: NativePlanQuestionCall) => { c.unansweredQuestionIndices = [0]; }, + (c: NativePlanQuestionCall) => { c.questions[0]!.multiSelect = true; }, + (c: NativePlanQuestionCall) => { c.questions.push(structuredClone(c.questions[0]!)); }, + (c: NativePlanQuestionCall) => { c.questions[0]!.options.push(structuredClone(c.questions[0]!.options[0]!)); }, + (c: NativePlanQuestionCall) => { c.questions[0]!.options[1]!.label = c.questions[0]!.options[0]!.label; }, + (c: NativePlanQuestionCall) => { c.answers = {}; }, + (c: NativePlanQuestionCall) => { c.answers = {[c.questions[0]!.question]: 'Foreign answer'}; }, + ]) { const c = fresh(); mutate(c); expect(first(c)).toBe(false); } + expect(engFirstReviewAUQ({...fp(fresh()), signature: 'foreign:call'})).toBe(false); + expect(engFirstReviewAUQ({...fp(fresh()), nativeCall: undefined})).toBe(false); + expect(engFirstReviewAUQ({...fp(fresh()), options: []})).toBe(false); + const mismatch = fp(fresh()); mismatch.options[0]!.label = 'foreign'; expect(engFirstReviewAUQ(mismatch)).toBe(false); + const wrongIndex = fp(fresh()); wrongIndex.options[0]!.index = 2; expect(engFirstReviewAUQ(wrongIndex)).toBe(false); + }); + + test('setup, foreign, hypothetical, quoted and mixed questions cannot open review', () => { + for (const header of ['Scope', 'Approach', 'Next review', 'Onboarding']) { + const c = fresh(); c.questions[0]!.header = header; expect(first(c)).toBe(false); + } + for (const transform of [ + (s: string) => s.replace('Architecture:', 'Approach:'), + (s: string) => s.replace('How should AuthBroker', 'If needed, how should AuthBroker'), + (s: string) => s.replace('AuthBroker access', 'the whole plan access'), + (s: string) => s.replace('plan-eng-cache-binding', 'plan-eng-setup'), + (s: string) => s.replace('plan-eng-cache-binding', 'foreign-cache-binding'), + (s: string) => s.replace(/ ]+>/, ''), + (s: string) => s + ' ', + (s: string) => s + ' Approve the release too.', + (s: string) => '> ' + s, + (s: string) => '```text\n' + s + '\n```', + ]) expect(first(question(fresh(), transform))).toBe(false); + }); + + test('both descriptions must affirm the existing dependency and remedy without extra obligations', () => { + for (const [index, transform] of [ + [0, (s: string) => s.replace('Eliminates module-level mutable state entirely.', 'Does not eliminate module-level mutable state.')], + [0, (s: string) => s.replace('Eliminates module-level mutable state entirely.', 'If shared state exists, eliminates it.')], + [0, (s: string) => s.replace('AuthBroker receives', 'DifferentComponent receives')], + [1, (s: string) => s.replace('same pattern as the current plan', 'unlike the current plan')], + [1, (s: string) => s.replace('makes tests require module-level mocking', 'does not make tests require module-level mocking')], + [1, (s: string) => s.replace('AuthBroker imports', 'DifferentComponent imports')], + [0, (s: string) => s + ' Also grant every tenant access.'], + [1, (s: string) => s + ' Also approve the missing timeout policy.'], + [0, (s: string) => '> ' + s], + [1, (s: string) => '```text\n' + s + '\n```'], + ] as const) { + const c = fresh(); const option = c.questions[0]!.options[index]!; + option.description = transform(option.description ?? ''); expect(first(c)).toBe(false); + } + const c = fresh(); c.questions[0]!.options[0]!.label = 'Run /office-hours'; + c.answers = {[c.questions[0]!.question]: 'Run /office-hours'}; expect(first(c)).toBe(false); + }); +}); diff --git a/test/fixtures/autoplan-setup-z-packet.json b/test/fixtures/autoplan-setup-z-packet.json new file mode 100644 index 000000000..92da11284 --- /dev/null +++ b/test/fixtures/autoplan-setup-z-packet.json @@ -0,0 +1,41 @@ +{ + "pendingCall": { + "sessionId": "fdd778bd-7b37-48e2-a9ef-8a23236eb739", + "toolUseId": "toolu_01PCZUJyfaiLWEa1TsnuffPp", + "questions": [ + { + "question": "D1 \u2014 Add gstack skill routing rules to CLAUDE.md? ", + "header": "Routing rules", + "multiSelect": false, + "options": [ + { + "label": "Add routing rules (Recommended)", + "description": "Appends a skill routing section to CLAUDE.md so future sessions automatically invoke the right skill (e.g. /autoplan for reviews, /ship for deploys) without you needing to type the command each time. One-time setup per project." + }, + { + "label": "Skip, I'll invoke manually", + "description": "No change to CLAUDE.md. You'll continue calling skills yourself with /skill-name as you do now." + } + ] + }, + { + "question": "D2 \u2014 No design doc found for this branch. Run /office-hours first to sharpen the review input?", + "header": "Design doc", + "multiSelect": false, + "options": [ + { + "label": "Skip, proceed with review", + "description": "Start the full CEO \u2192 Design \u2192 DX \u2192 Eng review pipeline now using the plan as-is. Recommended when the plan context is already rich enough. (Recommended)" + }, + { + "label": "Run /office-hours first", + "description": "Produces a structured problem statement, premise challenge, and explored alternatives before the review. Takes ~10 min. Gives the review sharper, better-grounded input." + } + ] + } + ], + "answered": false, + "failed": false + }, + "screen": " bun -e 'console.log(require(\"fs\").realpathSync(process.argv[1]))'\n \"$HOME/.claude/skills/gstack/bin/gstack-autoplan-snapshot.ts\" 2>/dev/n\u2026)\n \u23bf \u00a0/home/vercel-sandbox/gstack/bin/gstack-autoplan-snapshot.ts\n RESTORE_PATH=/tmp/gstack-paid-shard-VpofCc/tmp/gstack-hermetic-2786306-ZTxUr3/skill-home-SKSZlS/.gstack/projec\n ts/gstack-autoplan-chain-OmAEGg/main-autoplan-restore-20260909-120209.md\n \u2026 +3 lines (ctrl+o to expand)\n \u23bf \u00a0Allowed by auto mode classifier\n\n\u25cf Bash(SNAPSHOT_TOOL=\"/home/vercel-sandbox/gstack/bin/gstack-autoplan-snapshot.ts\"\n SOURCE_PLAN=\"/tmp/gstack-paid-shard-VpofCc/tmp/gstack-autoplan-chain-OmAEGg/.claude/\u2026)\n \u23bf \u00a0{\n \"sourcePlan\": \"/tmp/gstack-paid-shard-VpofCc/tmp/gstack-autoplan-chain-OmAEGg/.claude/plans/ui-heavy-feature\n .md\",\n \u2026 +57 lines (ctrl+o to expand)\n \u23bf \u00a0Allowed by auto mode classifier\n\n\u25cf Phase 0 initialized. Scope detected: UI scope (dashboard/component/modal/layout/button terms) and DX scope\n (API/endpoint/integration/implement terms). Phases 1, 2, 2.5, and 3 will all run.\n\n Now handling the GSTACK_INSTRUCTION routing block before proceeding:\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\nPlanning: /tmp/gstack-paid-shard-VpofCc/tmp/gstack-hermetic-2786306-ZTxUr3/with-skills/.claude/plans/expressive-twirling\n-wigderson.md \n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\u2190 \u2610 Routing rules \u2610 Design doc \u2714 Submit \u2192\n\nD1 \u2014 Add gstack skill routing rules to CLAUDE.md? \n\n\u276f 1. Add routing rules (Recommended)\n Appends a skill routing section to CLAUDE.md so future sessions automatically invoke the right skill (e.g.\n /autoplan for reviews, /ship for deploys) without you needing to type the command each time. One-time setup per\n project.\n 2. Skip, I'll invoke manually\n No change to CLAUDE.md. You'll continue calling skills yourself with /skill-name as you do now.\n 3. Type something.\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n 4. Chat about this\n\nEnter to select \u00b7 Tab/Arrow keys to navigate \u00b7 Esc to cancel\n" +} diff --git a/test/fixtures/ceo-approach-z-call.json b/test/fixtures/ceo-approach-z-call.json new file mode 100644 index 000000000..95e988e9b --- /dev/null +++ b/test/fixtures/ceo-approach-z-call.json @@ -0,0 +1,32 @@ +{ + "sessionId": "464da04e-2bf6-4d64-99c5-9914341b0493", + "toolUseId": "toolu_01ChdWcWJGU68XdpoR6qv18o", + "questions": [ + { + "question": "D3 \u2014 Which implementation approach should this plan follow?\nProject/branch: gstack-plan-count-kMoX7Y / main \u2014 new StripePaymentWebhookHandler\nELI10: The plan has a SQL injection bug baked in \u2014 it explicitly says an external string goes raw into a SQL query. That\u2019s a hard blocker regardless of approach. The question is what else to fix in the same pass: just the blocker, everything, or send the plan back for a full rewrite.\nStakes if we pick wrong: Approach A ships fast but leaves N+1 queries and no tests; approach C delays ship by days with no guarantee the rewrite is better.\nRecommendation: B because at CC+gstack speed, fixing all 5 issues costs ~15 minutes of implementation time versus ~5 for the minimal fix \u2014 the gap is negligible and the result is far more solid.\nCompleteness: A=7/10, B=10/10, C=8/10\nPros / cons:\nA) Minimal Fix: patch SQL injection + email error handling, defer N+1 and tests (recommended)\n \u2705 Fastest path to a secure, shippable handler \u2014 blocks the critical vulnerability immediately\n \u2705 Smallest diff; least risk of introducing new issues in the same PR\n \u274c N+1 queries remain and will create database load at payment burst volume\n \u274c No automated tests \u2014 next refactor touching shared utils could silently break payment handling\nB) Proper Integration: fix all 5 issues (SQL, email, N+1, tests, dispatcher) (recommended)\n \u2705 Fully secure and observable \u2014 parameterized queries, isolated email errors, eager-loaded orders\n \u2705 WebhookDispatcher integration means future webhooks get the same patterns automatically\n \u2705 Tests prevent silent regressions in shared utilities that touch this handler\n \u274c More scope than minimal; requires understanding WebhookDispatcher internals before integrating\nC) Reject and Redesign: return the plan for rewrite before any implementation\n \u2705 Forces author to internalize secure patterns from scratch \u2014 no bandaging a broken foundation\n \u2705 Cleanest outcome if the plan\u2019s author needs to own the security thinking themselves\n \u274c Delays ship by ~2 days (human) while the plan is rewritten\n \u274c No guarantee the rewrite arrives with better architecture without a structured redesign process\nNet: This is a tradeoff between implementation speed and completeness \u2014 B costs almost nothing extra in CC time, so \u201cminimal viable\u201d reasoning doesn\u2019t apply here the way it would in human-only development. ", + "header": "Approach", + "options": [ + { + "label": "A \u2014 Minimal Fix", + "description": "Fix SQL injection and email error handling only. Defer N+1 fix, tests, and dispatcher integration." + }, + { + "label": "B \u2014 Proper Integration (recommended)", + "description": "Fix all 5 issues: SQL injection, email handling, N+1 queries, tests, WebhookDispatcher integration." + }, + { + "label": "C \u2014 Reject and Redesign", + "description": "Return the plan for a full rewrite before any implementation proceeds." + } + ], + "multiSelect": false + } + ], + "answered": true, + "failed": false, + "answers": { + "D3 \u2014 Which implementation approach should this plan follow?\nProject/branch: gstack-plan-count-kMoX7Y / main \u2014 new StripePaymentWebhookHandler\nELI10: The plan has a SQL injection bug baked in \u2014 it explicitly says an external string goes raw into a SQL query. That\u2019s a hard blocker regardless of approach. The question is what else to fix in the same pass: just the blocker, everything, or send the plan back for a full rewrite.\nStakes if we pick wrong: Approach A ships fast but leaves N+1 queries and no tests; approach C delays ship by days with no guarantee the rewrite is better.\nRecommendation: B because at CC+gstack speed, fixing all 5 issues costs ~15 minutes of implementation time versus ~5 for the minimal fix \u2014 the gap is negligible and the result is far more solid.\nCompleteness: A=7/10, B=10/10, C=8/10\nPros / cons:\nA) Minimal Fix: patch SQL injection + email error handling, defer N+1 and tests (recommended)\n \u2705 Fastest path to a secure, shippable handler \u2014 blocks the critical vulnerability immediately\n \u2705 Smallest diff; least risk of introducing new issues in the same PR\n \u274c N+1 queries remain and will create database load at payment burst volume\n \u274c No automated tests \u2014 next refactor touching shared utils could silently break payment handling\nB) Proper Integration: fix all 5 issues (SQL, email, N+1, tests, dispatcher) (recommended)\n \u2705 Fully secure and observable \u2014 parameterized queries, isolated email errors, eager-loaded orders\n \u2705 WebhookDispatcher integration means future webhooks get the same patterns automatically\n \u2705 Tests prevent silent regressions in shared utilities that touch this handler\n \u274c More scope than minimal; requires understanding WebhookDispatcher internals before integrating\nC) Reject and Redesign: return the plan for rewrite before any implementation\n \u2705 Forces author to internalize secure patterns from scratch \u2014 no bandaging a broken foundation\n \u2705 Cleanest outcome if the plan\u2019s author needs to own the security thinking themselves\n \u274c Delays ship by ~2 days (human) while the plan is rewritten\n \u274c No guarantee the rewrite arrives with better architecture without a structured redesign process\nNet: This is a tradeoff between implementation speed and completeness \u2014 B costs almost nothing extra in CC time, so \u201cminimal viable\u201d reasoning doesn\u2019t apply here the way it would in human-only development. ": "A \u2014 Minimal Fix" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:05:36.962Z" +} diff --git a/test/fixtures/ceo-approach-z-screen.txt b/test/fixtures/ceo-approach-z-screen.txt new file mode 100644 index 000000000..6f38e1f0e --- /dev/null +++ b/test/fixtures/ceo-approach-z-screen.txt @@ -0,0 +1,39 @@ + ☐ Approach + +│ D3 — Which implementation approach should this plan follow? +│ Project/branch: gstack-plan-count-kMoX7Y / main — new StripePaymentWebhookHandler +│ ELI10: The plan has a SQL injection bug baked in — it explicitly says an external string goes raw into a SQL query. +│ That’s a hard blocker regardless of approach. The question is what else to fix in the same pass: just the blocker, +│ everything, or send the plan back for a full rewrite. +│ Stakes if we pick wrong: Approach A ships fast but leaves N+1 queries and no tests; approach C delays ship by days +│ with no guarantee the rewrite is better. +│ Recommendation: B because at CC+gstack speed, fixing all 5 issues costs ~15 minutes of implementation time versus ~5 +│ for the minimal fix — the gap is negligible and the result is far more solid. +│ Completeness: A=7/10, B=10/10, C=8/10 +│ Pros / cons: +│ A) Minimal Fix: patch SQL injection + email error handling, defer N+1 and tests (recommended) +│ ✅ Fastest path to a secure, shippable handler — blocks the critical vulnerability immediately +│ ✅ Smallest diff; least risk of introducing new issues in the same PR +│ ❌ N+1 queries remain and will create database load at payment burst volume +│ ❌ No automated tests — next refactor touching shared utils could silently break payment handling +│ B) Proper Integration: fix all 5 issues (SQL, email, N+1, tests, dispatcher) (recommended) +│ ✅ Fully secure and observable — parameterized queries, isolated email errors, eager-loaded orders +│ ✅ WebhookDispatcher integration means future webhooks get the same patterns automatically +│ ✅ Tests prevent silent regressions in shared utilities that touch this handler +│ ❌ More scope than minimal; requires understanding WebhookDispatcher internals before integrating +│ C) Reject and Redesign: return the plan for rewrite before any implementation +│ ✅ Forces author to internalize secure patterns from scratch — no bandaging a broken foundation +│ ✅ Cleanest outcome if the plan’s author needs to own the security thinking themselves +│ ❌ Dela… + +❯ 1. A — Minimal Fix + Fix SQL injection and email error handling only. Defer N+1 fix, tests, and dispatcher integration. + 2. B — Proper Integration (recommended) + Fix all 5 issues: SQL injection, email handling, N+1 queries, tests, WebhookDispatcher integration. + 3. C — Reject and Redesign + Return the plan for a full rewrite before any implementation proceeds. + 4. Type something. +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + 5. Chat about this + +Enter to select · ↑/↓ to navigate · Esc to cancel diff --git a/test/fixtures/ceo-handoff-z-call.json b/test/fixtures/ceo-handoff-z-call.json new file mode 100644 index 000000000..7416a6475 --- /dev/null +++ b/test/fixtures/ceo-handoff-z-call.json @@ -0,0 +1,221 @@ +{ + "calls": [ + { + "sessionId": "60a59c7e-1876-4b49-9ba1-30812b66f0f9", + "toolUseId": "toolu_01GbEV4Xaw2sSNEUZ69NLH7X", + "questions": [ + { + "question": "No design doc found for this branch. `/office-hours` produces a structured problem statement, premise challenge, and explored alternatives \u2014 it gives this review much sharper input to work with. Takes about 10 minutes. The design doc is per-feature, not per-product \u2014 it captures the thinking behind this specific change. Run /office-hours first, or skip and proceed with standard review?", + "header": "Design Doc", + "multiSelect": false, + "options": [ + { + "label": "Skip \u2014 proceed with standard review (Recommended)", + "description": "Jump straight into the CEO review using the plan as written. Best when you already have a clear problem statement and the plan is ready to be stress-tested." + }, + { + "label": "Run /office-hours first", + "description": "Run /office-hours to produce a structured problem statement and explore alternatives before the review. Recommended when scope or framing is still fuzzy." + } + ] + }, + { + "question": "gstack can search learnings from your other projects on this machine to find patterns that might apply here. This stays local \u2014 no data leaves your machine. Enable cross-project learnings?", + "header": "Learnings", + "multiSelect": false, + "options": [ + { + "label": "Enable cross-project learnings (Recommended)", + "description": "Search learnings across all gstack projects on this machine. Good for solo devs who want patterns to compound across work." + }, + { + "label": "Keep project-scoped only", + "description": "Only search learnings from this specific project. Better for consultants or teams where cross-project bleed would be a concern." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "No design doc found for this branch. `/office-hours` produces a structured problem statement, premise challenge, and explored alternatives \u2014 it gives this review much sharper input to work with. Takes about 10 minutes. The design doc is per-feature, not per-product \u2014 it captures the thinking behind this specific change. Run /office-hours first, or skip and proceed with standard review?": "Skip \u2014 proceed with standard review (Recommended)", + "gstack can search learnings from your other projects on this machine to find patterns that might apply here. This stays local \u2014 no data leaves your machine. Enable cross-project learnings?": "Enable cross-project learnings (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:02:43.039Z" + }, + { + "sessionId": "60a59c7e-1876-4b49-9ba1-30812b66f0f9", + "toolUseId": "toolu_019krXyfTgpZTLnETr41y6sn", + "questions": [ + { + "question": "D1 \u2014 Which implementation approach should these tests follow? ", + "header": "Approach", + "multiSelect": false, + "options": [ + { + "label": "B: Two tests + assertion hardening (Recommended)", + "description": "Happy path + 502 tests using existing factory/mock/sleeper, with explicit assertions on Stripe call count, exact backoff sequence, and receipt fields. Turns tests into contracts, not vibe checks. Completeness: 8/10. Effort: human ~1.5 hrs / CC ~45 min." + }, + { + "label": "A: Exactly the two tests as written", + "description": "Happy path and 502 retry-exhaustion, minimal scope. Good enough, but assertions may remain vague (e.g. 'correct receipt' without field-level verification). Completeness: 6/10. Effort: human ~1 hr / CC ~30 min." + }, + { + "label": "C: Parametric/table-driven test matrix", + "description": "Wrap both cases in a table-driven structure. Scales well but adds abstraction overhead for just two cases (YAGNI risk). Completeness: 8/10. Effort: human ~2-3 hrs / CC ~1 hr." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D1 \u2014 Which implementation approach should these tests follow? ": "B: Two tests + assertion hardening (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:03:36.648Z" + }, + { + "sessionId": "60a59c7e-1876-4b49-9ba1-30812b66f0f9", + "toolUseId": "toolu_01GxYSu6ScjDi32vVtTumWiy", + "questions": [ + { + "question": "D2 \u2014 How should I approach the review of this test plan? ", + "header": "Review Mode", + "multiSelect": false, + "options": [ + { + "label": "HOLD SCOPE \u2014 maximum rigor (Recommended)", + "description": "The two-test scope is right. Review for correctness, assertion completeness, edge cases, failure modes, and test isolation. Make it bulletproof. No expansions surfaced. Right call for a targeted test-coverage plan." + }, + { + "label": "SELECTIVE EXPANSION \u2014 hold scope but show options", + "description": "Review the two-test plan rigorously AND surface adjacent test opportunities (e.g. 429 retry path, receipt field-format variants, idempotency) as individual cherry-pick decisions. Neutral posture on each expansion." + }, + { + "label": "SCOPE REDUCTION \u2014 find the minimum", + "description": "Challenge whether both tests are needed now or one can be deferred. Ruthless minimum. Usually right for a hotfix; probably overkill here." + }, + { + "label": "SCOPE EXPANSION \u2014 think bigger", + "description": "Treat this as an opportunity to design the full test pyramid for processPayment(). Dream state: complete path coverage across all Stripe response codes, receipt formats, concurrency. Right when you want to build the cathedral now." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D2 \u2014 How should I approach the review of this test plan? ": "HOLD SCOPE \u2014 maximum rigor (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:03:50.683Z" + }, + { + "sessionId": "60a59c7e-1876-4b49-9ba1-30812b66f0f9", + "toolUseId": "toolu_017UUyrJxdy46LpFGbVdss4Y", + "questions": [ + { + "question": "D3 \u2014 Section 2 finding: the 502 exhaustion test needs a concrete failure contract. What does processPayment() do when retries are exhausted? ", + "header": "Failure contract", + "multiSelect": false, + "options": [ + { + "label": "Raises a specific exception (Recommended)", + "description": "processPayment() raises a named exception class (e.g. PaymentGatewayError, StripeChargeError) when retries are exhausted. The test asserts that exact exception is raised and that no receipt is generated. Plan should name the class. Most common in Ruby/Python codebases." + }, + { + "label": "Returns a failure Result/Error object", + "description": "processPayment() returns a Result or Response object with a success flag or error code (e.g. Result(ok: false, error: :gateway_unavailable)). The test asserts the error field value and that no receipt is in the return. Common in functional or Rust-influenced patterns." + }, + { + "label": "Leave as-is \u2014 let implementer decide", + "description": "Keep 'fails clean' as the stated outcome and let the implementer pick the assertion form. Defers the decision; risks an under-specified test that passes on wrong behavior (e.g. returns nil but no assertion on why)." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D3 \u2014 Section 2 finding: the 502 exhaustion test needs a concrete failure contract. What does processPayment() do when retries are exhausted? ": "Raises a specific exception (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:06:47.281Z" + }, + { + "sessionId": "60a59c7e-1876-4b49-9ba1-30812b66f0f9", + "toolUseId": "toolu_01S2jGhUn6178QyZPcBxN8bf", + "questions": [ + { + "question": "D4 \u2014 Section 5/6 finding: 'assert correct receipt is generated' is under-specified for Approach B. The test needs to assert specific receipt fields. How should the plan specify the receipt assertions? ", + "header": "Receipt assertions", + "multiSelect": false, + "options": [ + { + "label": "Name key receipt fields in the plan (Recommended)", + "description": "The plan lists the fields the test must assert: at minimum stripe_charge_id, amount, and currency. Any field whose value the test can know from the mock input is asserted explicitly. This turns 'correct receipt' into a verifiable contract. Implementer looks up exact field names from the Receipt class." + }, + { + "label": "Assert non-nil receipt with no zero/nil fields", + "description": "The test asserts that a receipt was returned and that its required fields are all present (non-nil, non-zero). Less precise than field-value matching, but catches the case where the receipt builder returns a blank receipt or nil." + }, + { + "label": "Leave as 'correct receipt generated' \u2014 implementer decides", + "description": "Keep the vague assertion. Risks a test that passes even if the wrong charge amount is in the receipt. Acceptable only if the receipt builder already has its own unit tests that verify field values (the plan says it does, but that's the builder, not the processPayment integration)." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D4 \u2014 Section 5/6 finding: 'assert correct receipt is generated' is under-specified for Approach B. The test needs to assert specific receipt fields. How should the plan specify the receipt assertions? ": "Name key receipt fields in the plan (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:07:11.375Z" + }, + { + "sessionId": "60a59c7e-1876-4b49-9ba1-30812b66f0f9", + "toolUseId": "toolu_01ALV58ua7wEKobSnGSNVjCF", + "questions": [ + { + "question": "D5 \u2014 CEO Review is CLEAR. Eng Review is the required shipping gate and hasn't run yet. What's next? ", + "header": "Next review", + "multiSelect": false, + "options": [ + { + "label": "Run /plan-eng-review next (Recommended)", + "description": "Architecture, code quality, test design, performance, and deployment review. The required gate before shipping. Run this before implementation begins to catch any structural issues in how the tests are wired up." + }, + { + "label": "Skip \u2014 I'll handle reviews manually", + "description": "Proceed to implementation directly. You can run /plan-eng-review later if needed. Eng Review is required before shipping but not before starting implementation." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D5 \u2014 CEO Review is CLEAR. Eng Review is the required shipping gate and hasn't run yet. What's next? ": "Run /plan-eng-review next (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:11:32.075Z" + } + ], + "planReadyRequests": [ + { + "sessionId": "60a59c7e-1876-4b49-9ba1-30812b66f0f9", + "toolUseId": "toolu_01VxXyEi8U4W88tgfEKmCrvJ", + "timestamp": "2026-09-09T12:11:48.064Z", + "failed": false, + "source": "pre_tool_use" + } + ], + "report": "# Payment Processing \u2014 Test Coverage\n\n## Context\n\n`processPayment()` has zero unit tests on its two most critical paths: success\n(correctness) and 502 retry-exhaustion (graceful degradation). A broken happy\npath or a retry that loops forever or that panics instead of failing cleanly\nare both silent production incidents under the current coverage. This plan adds\nexactly two unit tests to close that gap, using the existing test factory,\nStripe mock, and virtual sleeper already in place. Production code is not\ntouched.\n\nReview mode: **HOLD SCOPE**. Approach: **B \u2014 assertion hardening** (explicit\ncall count, specific receipt fields, named exception on failure). Approved via\nD1/D2 before review sections began.\n\n---\n\n## Scope\n\n### Two unit tests for `processPayment()`\n\n**Test 1 \u2014 Happy path**\n\nSetup:\n- `payment_test_factory(max_retries=1)` with Stripe mock configured to return\n HTTP 200 on the charge call.\n\nAssertions (Approach B \u2014 field-level, not presence-only):\n- Receipt is returned (non-nil).\n- `stripe_charge_id` matches the mock charge ID.\n- `amount` matches the input charge amount.\n- `currency` matches the input currency.\n- Stripe mock was called exactly **1 time** (no spurious retries on success).\n\n> Implementer: look up exact field names from the `Receipt` class before\n> writing assertions. The three fields above are the minimum; assert any\n> additional fields whose values are deterministic from the mock inputs.\n\n**Test 2 \u2014 502 exhaustion path**\n\nSetup:\n- `payment_test_factory(max_retries=1)` with Stripe mock configured to return\n HTTP 502 on **every** call.\n- Virtual sleeper injected (records backoff without real delays).\n\nAssertions (Approach B):\n- `processPayment()` raises a named exception (look up the exact class in the\n production code \u2014 e.g. `PaymentGatewayError` or `StripeChargeError`). The\n plan must name this class before implementation begins.\n- **No receipt is generated** on this path (anti-assertion: exception raised\n means no receipt returned).\n- Stripe mock was called exactly **2 times** (initial attempt + exactly one\n retry; `max_retries=1` enforced by the factory).\n- Virtual sleeper recorded **1 backoff event** between the two charge attempts.\n\n### Existing infrastructure retained\n\nThe following are reused as-is:\n- `payment_test_factory` (configures `max_retries=1`, exposes mock call\n history)\n- Stripe mock (records call history)\n- Virtual sleeper (records backoff sequence; no real delays)\n- Stripe adapter regression suite (network timeouts, 402 declines, 429 rate\n limits, 502\u2192200 recovery) \u2014 unchanged\n- Receipt-builder regression tests \u2014 unchanged\n\n---\n\n## Architecture\n\n```\nPRODUCTION (unchanged):\n processPayment()\n \u251c\u2500\u2500 Stripe HTTP client [charge \u2192 200 OK | 402 | 429 | 502]\n \u2514\u2500\u2500 Receipt builder [build receipt from charge_response]\n\nNEW UNIT TESTS:\n Test 1 (happy path)\n payment_test_factory(max_retries=1)\n \u2514\u2500\u2500 Stripe mock (returns 200) \u2500\u2500\u2192 processPayment()\n assert: receipt.stripe_charge_id \u2713\n receipt.amount \u2713\n receipt.currency \u2713\n mock.call_count == 1 \u2713\n\n Test 2 (502 exhaustion)\n payment_test_factory(max_retries=1)\n \u2514\u2500\u2500 Stripe mock (returns 502 \u00d7 2) \u2500\u2500\u2192 processPayment()\n virtual_sleeper records 1 backoff event\n assert: raises NamedExceptionClass \u2713\n no receipt returned \u2713\n mock.call_count == 2 \u2713\n virtual_sleeper.events.count == 1 \u2713\n\nData flow \u2014 502 path:\n charge_args \u2500\u2500\u25b6 processPayment() \u2500\u2500\u25b6 Stripe mock (502)\n \u2502\n virtual_sleeper.record_backoff()\n \u2502\n retry \u2500\u2500\u25b6 Stripe mock (502)\n \u2502\n raises NamedExceptionClass\n (no receipt generated)\n```\n\n---\n\n## NOT in scope\n\n- `processPayment()` production code changes \u2014 explicitly excluded.\n- Additional error code paths (402 decline, 429 rate limit, 503, 504, 0/timeout)\n as unit tests for `processPayment()` \u2014 covered at the adapter level; deferred\n from this plan.\n- Parametric/table-driven test structure \u2014 YAGNI for two cases; consider when\n adding a 3rd path.\n- Idempotency testing \u2014 separate concern; no infrastructure for it yet.\n- Receipt field-value tests for every possible field \u2014 three key fields minimum;\n implementer adds more if deterministic from mock inputs.\n\n## What already exists\n\n| Existing piece | Solves which sub-problem | Reused? |\n|----------------|--------------------------|---------|\n| `payment_test_factory` | Stripe mock wiring, `max_retries=1`, call history | \u2713 yes |\n| Virtual sleeper | Backoff recording without real delays | \u2713 yes |\n| Stripe mock (call history API) | Asserting exact call count | \u2713 yes |\n| Stripe adapter suite (502\u2192200 recovery) | Partial coverage of the retry path | \u2713 retained (not replaced) |\n| Receipt-builder regression tests | Receipt field correctness | \u2713 retained (not replaced) |\n\n## Dream state delta\n\n```\nCURRENT STATE THIS PLAN 12-MONTH IDEAL\nNo unit tests on Two unit tests: Full path coverage:\nprocessPayment(). happy + 502 exhaustion happy, 402, 429, 503,\nStripe adapter suite with assertion 504, 0/timeout, partial\ncovers adapter-level hardening (call count, charges, idempotency,\nedge cases only. receipt fields, concurrency. Test\nprocessPayment()'s own named exception). pyramid complete.\ncontract is untested.\n```\n\n---\n\n## Error & Rescue Registry\n\n| Method / Codepath | What can go wrong | Exception class | Rescued? | Rescue action | User sees (test asserts) |\n|---|---|---|---|---|---|\n| processPayment() \u2014 happy | Stripe returns 200 | N/A | N/A | \u2014 | receipt with correct fields |\n| processPayment() \u2014 attempt 1 | Stripe returns 502 | NamedExceptionClass (TBD) | Y | backoff, retry | nothing yet |\n| processPayment() \u2014 attempt 2 (retry) | Stripe returns 502 | NamedExceptionClass (TBD) | Y | retry exhausted \u2192 raise | exception raised; no receipt |\n\n> **Action required before implementation:** identify the exact exception class\n> raised by `processPayment()` on 502 retry exhaustion (approved D3). Update\n> the plan with the class name before writing the test.\n\n## Failure Modes Registry\n\n| Codepath | Failure mode | Rescued? | Tested? | User sees | Logged? |\n|---|---|---|---|---|---|\n| processPayment() happy | Stripe 200, receipt built | N/A | \u2713 (Test 1) | correct receipt | N/A |\n| processPayment() 502 (attempt 1) | Stripe 502 | Y | \u2713 (Test 2) | nothing (transparent retry) | implied |\n| processPayment() 502 (attempt 2) | Retry exhausted | Y (raises) | \u2713 (Test 2) | named exception | implied |\n| Receipt builder fails | BuildError | handled separately | \u2713 (existing regression) | \u2014 | existing |\n\nNo CRITICAL GAPSs remaining after D3 and D4 approvals.\n\n---\n\n## Implementation Tasks\n\nSynthesized from this review's findings. Run with Claude Code; checkbox as you ship.\n\n- [ ] **T1 (P1, human: ~30min / CC: ~5min)** \u2014 processPayment() test \u2014 Specify and assert the named exception class on 502 retry exhaustion\n - Surfaced by: Section 2 Error & Rescue Map \u2014 \"fails clean\" was undefined; D3 approved raising a named exception\n - Files: `payment_test.rb` (or equivalent)\n - Verify: Test passes with the exception assertion; fails if you remove the retry or change the exception class\n\n- [ ] **T2 (P1, human: ~20min / CC: ~5min)** \u2014 processPayment() test \u2014 Assert specific receipt fields in happy path test\n - Surfaced by: Section 5/6 Code Quality / Test Review \u2014 \"correct receipt is generated\" was vague; D4 approved field-level assertions\n - Files: `payment_test.rb`\n - Verify: Test passes with correct mock; fails if amount or stripe_charge_id are wrong\n\n- [ ] **T3 (P1, human: ~10min / CC: ~2min)** \u2014 processPayment() test \u2014 Add anti-assertion confirming no receipt is generated on 502 exhaustion\n - Surfaced by: Section 6 Test Review \u2014 502 path must confirm exception path does not accidentally return a receipt\n - Files: `payment_test.rb`\n - Verify: Test would fail if processPayment() returned a receipt alongside the exception\n\n---\n\n```\n+====================================================================+\n| MEGA PLAN REVIEW \u2014 COMPLETION SUMMARY |\n+====================================================================+\n| Mode selected | HOLD SCOPE |\n| System Audit | Single-commit fixture repo; test-only plan |\n| Step 0 | Approach B approved (D1); HOLD SCOPE (D2) |\n| Section 1 (Arch) | 0 issues found |\n| Section 2 (Errors) | 2 paths mapped, 0 GAPS after D3 approval |\n| Section 3 (Security)| 0 issues found |\n| Section 4 (Data/UX) | 0 additional edge cases unhandled |\n| Section 5 (Quality) | 1 issue found (receipt spec, resolved D4) |\n| Section 6 (Tests) | Diagram produced, 0 gaps after D3+D4 |\n| Section 7 (Perf) | 0 issues found |\n| Section 8 (Observ) | 0 gaps found |\n| Section 9 (Deploy) | 0 risks flagged |\n| Section 10 (Future) | Reversibility: 5/5, debt items: 0 |\n| Section 11 (Design) | SKIPPED (no UI scope) |\n+--------------------------------------------------------------------+\n| NOT in scope | written (5 items) |\n| What already exists | written (5 pieces reused) |\n| Dream state delta | written |\n| Error/rescue registry| 3 paths, 0 CRITICAL GAPS |\n| Failure modes | 4 total, 0 CRITICAL GAPS |\n| TODOS.md updates | 0 items proposed (HOLD SCOPE; no deferred gaps)|\n| Scope proposals | 0 proposed, 0 accepted (HOLD SCOPE) |\n| CEO plan | skipped (HOLD SCOPE) |\n| Outside voice | skipped (codex_reviews disabled) |\n| Lake Score | 2/2 recommendations chose complete option |\n| Diagrams produced | 2 (system arch + data flow) |\n| Stale diagrams found | 0 |\n| Unresolved decisions | 0 |\n+====================================================================+\n```\n\n---\n\n## GSTACK REVIEW REPORT\n\n| Review | Trigger | Why | Runs | Status | Findings |\n|--------|---------|-----|------|--------|----------|\n| CEO Review | `/plan-ceo-review` | Scope & strategy | 1 | CLEAN | mode: HOLD_SCOPE, 0 critical gaps; 2 assertion gaps resolved (D3, D4) |\n| Outside Review | disabled | Independent 2nd opinion | 0 | SKIPPED | codex_reviews=disabled |\n| Eng Review | `/plan-eng-review` | Architecture & tests (required) | 0 | \u2014 | not yet run |\n| Design Review | `/plan-design-review` | UI/UX gaps | 0 | \u2014 | no UI scope |\n| DX Review | `/plan-devex-review` | Developer experience gaps | 0 | \u2014 | not yet run |\n\n**OUTSIDE COVERAGE:** codex_reviews disabled; no outside voice run. Re-enable: `gstack-config set codex_reviews enabled`.\n\n**VERDICT:** CEO CLEARED \u2014 0 unresolved decisions, 0 critical gaps. Eng review required before shipping.\n\nNO UNRESOLVED DECISIONS\n", + "reportOriginalMtimeNs": "1788955804222902951", + "startedAt": 1788955258827, + "reportSha256": "338cf49934f58f341680d4617dbc28269264bcd95df1a40d3f31f0c14ab0450b" +} diff --git a/test/fixtures/design-gap-z-calls.json b/test/fixtures/design-gap-z-calls.json new file mode 100644 index 000000000..f717f875b --- /dev/null +++ b/test/fixtures/design-gap-z-calls.json @@ -0,0 +1,236 @@ +{ + "calls": [ + { + "sessionId": "775fbf0c-b6aa-4363-9927-31d7fc15f6d4", + "toolUseId": "toolu_01N4SZ3Q3XtSxJn6UnBvskFb", + "questions": [ + { + "question": "Gap 1 of 8 \u2014 Save button visual hierarchy: apply DESIGN.md primary button style?", + "header": "Gap 1: Button", + "multiSelect": false, + "options": [ + { + "label": "Apply DESIGN.md fix (recommended)", + "description": "Add to plan: Save gets #1d4ed8 filled + white text (primary); Reset, Cancel, Export get neutral ghost style. Closes the visual hierarchy gap exactly as DESIGN.md specifies. Implementation task T1 becomes committed." + }, + { + "label": "Defer to implementation", + "description": "Leave the gap named but unresolved. Engineer decides the button styles at implementation time without a spec. Risk: inconsistency with the design system or re-work after review." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "Gap 1 of 8 \u2014 Save button visual hierarchy: apply DESIGN.md primary button style?": "Apply DESIGN.md fix (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:16:44.618Z" + }, + { + "sessionId": "775fbf0c-b6aa-4363-9927-31d7fc15f6d4", + "toolUseId": "toolu_017zBFd8ifxP7MFxNBAC2TpV", + "questions": [ + { + "question": "Gap 2 of 8 \u2014 Spacing: standardize to DESIGN.md 8px-base scale?", + "header": "Gap 2: Spacing", + "multiSelect": false, + "options": [ + { + "label": "Apply DESIGN.md fix (recommended)", + "description": "Add to plan: sections 32px apart, field groups 24px, label-to-input 8px. Removes the 16px outlier and gives implementation a single source of truth. Implementation task T2 becomes committed." + }, + { + "label": "Defer to implementation", + "description": "Leave the inconsistency noted. Engineer picks spacing values during build. Risk: the 16px outlier persists and the form feels uneven at different scroll positions." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "Gap 2 of 8 \u2014 Spacing: standardize to DESIGN.md 8px-base scale?": "Apply DESIGN.md fix (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:17:07.703Z" + }, + { + "sessionId": "775fbf0c-b6aa-4363-9927-31d7fc15f6d4", + "toolUseId": "toolu_018S3TMnN9HV9S75z8svkcUN", + "questions": [ + { + "question": "Gap 3 of 8 \u2014 Error color contrast: apply DESIGN.md #991b1b on #fef2f2 tokens?", + "header": "Gap 3: Color", + "multiSelect": false, + "options": [ + { + "label": "Apply DESIGN.md fix (recommended)", + "description": "Add to plan: error text uses #991b1b on #fef2f2 surface with an icon. Brings error states to WCAG AA contrast (from the current ~3:1). Implementation task T3 becomes committed." + }, + { + "label": "Defer to implementation", + "description": "Leave the contrast gap noted. Engineer resolves the color at build time. Risk: a shipped accessibility violation, since the current ~3:1 is below WCAG AA minimum of 4.5:1." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "Gap 3 of 8 \u2014 Error color contrast: apply DESIGN.md #991b1b on #fef2f2 tokens?": "Apply DESIGN.md fix (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:17:24.757Z" + }, + { + "sessionId": "775fbf0c-b6aa-4363-9927-31d7fc15f6d4", + "toolUseId": "toolu_011rznHG5hrLyBFvZ4gCmt5s", + "questions": [ + { + "question": "Gap 4 of 8 \u2014 Typography: consolidate to two sizes per DESIGN.md (16px + 20px)?", + "header": "Gap 4: Type", + "multiSelect": false, + "options": [ + { + "label": "Apply DESIGN.md fix (recommended)", + "description": "Add to plan: all body text, form labels, and helper text use 16px; section headings (h2) use 20px. The rogue 14px and 18px sizes are removed. Creates stronger two-level hierarchy. Task T4 committed." + }, + { + "label": "Defer to implementation", + "description": "Leave the three-size inconsistency noted. Engineer picks which sizes to use at build time. Risk: the 14px text may persist and fail WCAG AA body-text contrast at small size." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "Gap 4 of 8 \u2014 Typography: consolidate to two sizes per DESIGN.md (16px + 20px)?": "Apply DESIGN.md fix (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:17:34.295Z" + }, + { + "sessionId": "775fbf0c-b6aa-4363-9927-31d7fc15f6d4", + "toolUseId": "toolu_01BWGpyri31dDYV7Qn1ntbrq", + "questions": [ + { + "question": "Gap 5 of 8 \u2014 Motion: add DESIGN.md spinner pattern for Save pending state?", + "header": "Gap 5: Motion", + "multiSelect": false, + "options": [ + { + "label": "Apply DESIGN.md fix (recommended)", + "description": "Add to plan: on Save click, disable the button, show an inline spinner beside 'Saving\u2026', set aria-busy=true. With reduced-motion: no spinner, text only. Success clears to 'Saved at HH:mm'. Task T5 committed." + }, + { + "label": "Defer to implementation", + "description": "Leave the frozen-page gap noted. Engineer designs the loading state at build time. Risk: inconsistency with the established spinner pattern and a 2-5s blank page on every save." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "Gap 5 of 8 \u2014 Motion: add DESIGN.md spinner pattern for Save pending state?": "Apply DESIGN.md fix (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:17:43.823Z" + }, + { + "sessionId": "775fbf0c-b6aa-4363-9927-31d7fc15f6d4", + "toolUseId": "toolu_01LngzQi1Ukw4EmxesxaSfDc", + "questions": [ + { + "question": "Gap 6 of 8 \u2014 Export-while-dirty: specify that Export reads saved values only?", + "header": "Gap 6: Export", + "multiSelect": false, + "options": [ + { + "label": "Export reads last-saved values (recommended)", + "description": "Add to plan: Export downloads the last-saved preferences, ignoring unsaved edits. No implicit save triggered. Unsaved-changes status persists. Matches the atomic-save philosophy already in the plan." + }, + { + "label": "Export reads current (unsaved) field values", + "description": "Add to plan: Export includes whatever is currently in the form fields, whether saved or not. The download reflects the user's in-progress state. Simpler to implement but may surprise users who export before saving." + }, + { + "label": "Defer to implementation", + "description": "Leave this scenario unspecified. Engineer decides at build time. Risk: inconsistent behavior between implementations or user confusion when exported JSON differs from what they see in the form." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "Gap 6 of 8 \u2014 Export-while-dirty: specify that Export reads saved values only?": "Export reads last-saved values (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:17:59.395Z" + }, + { + "sessionId": "775fbf0c-b6aa-4363-9927-31d7fc15f6d4", + "toolUseId": "toolu_01G1HXr6nAGLxkd2ntnqGpk7", + "questions": [ + { + "question": "Gap 7 of 8 \u2014 Ghost button spec: reference existing component or add explicit values?", + "header": "Gap 7: Ghost btn", + "multiSelect": false, + "options": [ + { + "label": "Reference existing Button component ghost variant (recommended)", + "description": "Add to plan: 'Reset, Cancel, Export use the existing Button component ghost variant. No new CSS values are introduced.' Clean, DRY, ties to the component system." + }, + { + "label": "State explicit CSS values in the plan", + "description": "Add to plan: ghost buttons have transparent background, 1px border #d1d5db, text #374151, hover background #f9fafb. Useful if the Button component doesn't yet have a ghost variant or its values are unclear." + }, + { + "label": "Defer \u2014 leave ghost style unspecified", + "description": "Let the engineer look up or invent ghost button values at build time. Risk: visual inconsistency with other ghost buttons in the app." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "Gap 7 of 8 \u2014 Ghost button spec: reference existing component or add explicit values?": "Reference existing Button component ghost variant (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:18:20.487Z" + }, + { + "sessionId": "775fbf0c-b6aa-4363-9927-31d7fc15f6d4", + "toolUseId": "toolu_01L28TR9p37GzvWFrLsMPZbH", + "questions": [ + { + "question": "Gap 8 of 8 \u2014 InlineStatus during validation: specify it reads 'Unsaved changes'?", + "header": "Gap 8: LiveRegion", + "multiSelect": false, + "options": [ + { + "label": "Specify 'Unsaved changes' during validation (recommended)", + "description": "Add to plan: when validation errors fire, InlineStatus text is 'Unsaved changes' (the normal dirty state). The live region announces once; field errors and ErrorSummary announce separately via focus. No simultaneous conflicting announcements." + }, + { + "label": "Defer \u2014 leave live region coordination unspecified", + "description": "Let the engineer decide what InlineStatus reads during validation. Risk: screen reader announces conflicting states simultaneously if the live region and error summary fire at the same moment." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "Gap 8 of 8 \u2014 InlineStatus during validation: specify it reads 'Unsaved changes'?": "Specify 'Unsaved changes' during validation (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:18:41.580Z" + } + ] +} diff --git a/test/fixtures/devex-count-z-calls.json b/test/fixtures/devex-count-z-calls.json new file mode 100644 index 000000000..47d0debc4 --- /dev/null +++ b/test/fixtures/devex-count-z-calls.json @@ -0,0 +1,258 @@ +[ + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_01YYoD4nRXm4BfiePigvsrN6", + "questions": [ + { + "question": "D1 \u2014 Does this empathy narrative match your ML engineer developer's actual experience?\n\nPersona: ML engineer, Python daily, terminal-oriented, target TTHW < 2 min\nMode: DX POLISH (pre-settled)\n\nI traced the actual path from README.md. Here's what your developer experiences today:\n\n> I install evalkit==2.0.0b1, set EVALKIT_API_KEY. The README says to run\n> `python examples/first_eval.py`. I try it:\n>\n> ```\n> python: can't open file 'examples/first_eval.py': [Errno 2] No such file or directory\n> ```\n>\n> The file is not in the published package (confirmed: docs/package-contents.txt).\n> After some confusion I find the demo command. I run `python -m evalkit.demo`.\n> For five minutes I watch: \"Waiting for CI check: 90s elapsed of 300s...\".\n> No explanation of why this check runs locally. Then: scores appear.\n> Total time: 6-7 min. First command failed. I'm not confident in this tool.\n\nI found 5 friction points. Settled decisions (persona, DX POLISH mode, terminal demo, benchmark) are not re-litigated \u2014 I'll go straight to the issues. ", + "header": "Narrative", + "multiSelect": false, + "options": [ + { + "label": "Accurate \u2014 proceed to issues (Recommended)", + "description": "This is accurate. Proceed to walk through the 5 friction points one at a time." + }, + { + "label": "Some details wrong", + "description": "The general picture is right but some specifics are off \u2014 describe corrections." + }, + { + "label": "Way off", + "description": "The actual getting-started experience is significantly different from this \u2014 describe what's actually happening." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D1 \u2014 Does this empathy narrative match your ML engineer developer's actual experience?\n\nPersona: ML engineer, Python daily, terminal-oriented, target TTHW < 2 min\nMode: DX POLISH (pre-settled)\n\nI traced the actual path from README.md. Here's what your developer experiences today:\n\n> I install evalkit==2.0.0b1, set EVALKIT_API_KEY. The README says to run\n> `python examples/first_eval.py`. I try it:\n>\n> ```\n> python: can't open file 'examples/first_eval.py': [Errno 2] No such file or directory\n> ```\n>\n> The file is not in the published package (confirmed: docs/package-contents.txt).\n> After some confusion I find the demo command. I run `python -m evalkit.demo`.\n> For five minutes I watch: \"Waiting for CI check: 90s elapsed of 300s...\".\n> No explanation of why this check runs locally. Then: scores appear.\n> Total time: 6-7 min. First command failed. I'm not confident in this tool.\n\nI found 5 friction points. Settled decisions (persona, DX POLISH mode, terminal demo, benchmark) are not re-litigated \u2014 I'll go straight to the issues. ": "Accurate \u2014 proceed to issues (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:05:40.860Z" + }, + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_01Hi3US42PswPmBjW2NvsUhi", + "questions": [ + { + "question": "D2 \u2014 Friction point 1: `examples/first_eval.py` is referenced in README but doesn't exist in the published package.\n\nPlan file: README.md line 11: \"follow the quickstart's command: `python examples/first_eval.py`\"\nEvidence: docs/package-contents.txt explicitly states: \"The quickstart references examples/first_eval.py, but that file is absent from both the published package and the release examples archive.\"\n\nYour ML engineer developer hits a FileNotFoundError on their very first command. This violates DX First Principle #1 (Zero friction at T0) and is the most visible broken contract in the release. The terminal demo (`python -m evalkit.demo`) is already in the package and works \u2014 it just needs to be the primary getting-started command.\n\nRecommendation: A \u2014 update README to point to `python -m evalkit.demo` as the canonical first command. It's already documented as the chosen delivery vehicle; the README just hasn't been updated to match.\nCompleteness: A=9/10, B=7/10, C=2/10 ", + "header": "FP1 Quickstart", + "multiSelect": false, + "options": [ + { + "label": "Fix in plan: update README to use python -m evalkit.demo (Recommended)", + "description": "Remove the reference to examples/first_eval.py from README. Replace with python -m evalkit.demo as the canonical first-run command. Completeness: 9/10." + }, + { + "label": "Fix in plan: include the missing file in the release", + "description": "Add examples/first_eval.py to the published package. Requires writing and testing the file before release. Completeness: 7/10." + }, + { + "label": "Acceptable \u2014 skip", + "description": "Leave the broken reference in place. Completeness: 2/10 \u2014 first command fails for every developer following the README." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D2 \u2014 Friction point 1: `examples/first_eval.py` is referenced in README but doesn't exist in the published package.\n\nPlan file: README.md line 11: \"follow the quickstart's command: `python examples/first_eval.py`\"\nEvidence: docs/package-contents.txt explicitly states: \"The quickstart references examples/first_eval.py, but that file is absent from both the published package and the release examples archive.\"\n\nYour ML engineer developer hits a FileNotFoundError on their very first command. This violates DX First Principle #1 (Zero friction at T0) and is the most visible broken contract in the release. The terminal demo (`python -m evalkit.demo`) is already in the package and works \u2014 it just needs to be the primary getting-started command.\n\nRecommendation: A \u2014 update README to point to `python -m evalkit.demo` as the canonical first command. It's already documented as the chosen delivery vehicle; the README just hasn't been updated to match.\nCompleteness: A=9/10, B=7/10, C=2/10 ": "Fix in plan: update README to use python -m evalkit.demo (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:05:54.895Z" + }, + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_01J4HW3Qwn9KWXkCgRHhPtEn", + "questions": [ + { + "question": "D3 \u2014 Friction point 2: mandatory 5-minute CI wait on first run, no skip flag.\n\nEvidence: docs/current-contracts.md: \"On a developer's first local evaluation, the SDK requires a successful remote CI check and blocks for five minutes before returning a result. There is no skip flag or offline first-run path.\"\nEvidence: docs/benchmarks.md: \"EvalKit took 6 minutes... The agreed target is under 2 minutes.\"\n\nYour ML engineer wants a local result first, before hooking up CI. The 5-minute mandatory wait IS the entire 4-minute gap between current TTHW (6 min) and the agreed target (< 2 min). No other single change closes this gap.\n\nThe plan retains this gate as-is. DX POLISH mode means working within existing release scope \u2014 but a gap this large between stated target and actual behavior is a contract worth calling out explicitly in the plan, even if the full fix isn't in scope.\n\nRecommendation: B \u2014 document the TTHW gap prominently in the plan and add a note that a --skip-ci-check flag is a prerequisite for hitting the < 2 min target. No scope addition now; P1 blocker for a follow-on.\nCompleteness: A=10/10, B=8/10, C=4/10 ", + "header": "FP2 CI wait", + "multiSelect": false, + "options": [ + { + "label": "Add --skip-ci-check to scope (Recommended if target matters)", + "description": "Add a --skip-ci-check or EVALKIT_SKIP_CI=1 flag to this release. Required to hit the < 2 min TTHW target. Completeness: 10/10." + }, + { + "label": "Document gap in plan, flag as P1 follow-up (Recommended for DX POLISH)", + "description": "Keep the 5-min wait. Add an explicit note in the plan that the < 2 min target is currently unachievable and a --skip-ci-check flag is the prerequisite. Completeness: 8/10." + }, + { + "label": "Retain as-is, no change", + "description": "Keep the mandatory wait and don't call it out. The stated TTHW target remains unmet without explanation. Completeness: 4/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D3 \u2014 Friction point 2: mandatory 5-minute CI wait on first run, no skip flag.\n\nEvidence: docs/current-contracts.md: \"On a developer's first local evaluation, the SDK requires a successful remote CI check and blocks for five minutes before returning a result. There is no skip flag or offline first-run path.\"\nEvidence: docs/benchmarks.md: \"EvalKit took 6 minutes... The agreed target is under 2 minutes.\"\n\nYour ML engineer wants a local result first, before hooking up CI. The 5-minute mandatory wait IS the entire 4-minute gap between current TTHW (6 min) and the agreed target (< 2 min). No other single change closes this gap.\n\nThe plan retains this gate as-is. DX POLISH mode means working within existing release scope \u2014 but a gap this large between stated target and actual behavior is a contract worth calling out explicitly in the plan, even if the full fix isn't in scope.\n\nRecommendation: B \u2014 document the TTHW gap prominently in the plan and add a note that a --skip-ci-check flag is a prerequisite for hitting the < 2 min target. No scope addition now; P1 blocker for a follow-on.\nCompleteness: A=10/10, B=8/10, C=4/10 ": "Add --skip-ci-check to scope (Recommended if target matters)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:06:10.945Z" + }, + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_016Kwybn1nSkZJvdeGXbZDmp", + "questions": [ + { + "question": "D4 \u2014 Friction point 3: `AuthError(\"request failed\")` tells the developer nothing useful.\n\nEvidence: docs/api.md: \"For an invalid API key, the SDK raises AuthError('request failed'). There is no authentication error code, explanation of the cause, or instruction for replacing the key. The plan retains this message.\"\nEvidence: docs/current-contracts.md: \"Authentication errors behave exactly as documented in docs/api.md. All other errors already identify the cause, relevant argument or file, and an actionable fix.\"\n\nThis is the only error that doesn't follow the pattern your other errors already follow. Your ML engineer sets a wrong API key and sees:\n AuthError: request failed\n\nThey don't know: was it a network error? A bad key? A rate limit? They have no idea what to fix. The Hall of Fame Tier 1 standard: problem + cause + fix. You're at Tier 3 (just a message string).\n\nA minimal fix: `AuthError(\"invalid API key \u2014 check EVALKIT_API_KEY or regenerate at [dashboard]\")`\n\nRecommendation: A \u2014 update the error message. It's a one-line change. Completeness: A=9/10, B=6/10, C=3/10 ", + "header": "FP3 AuthError", + "multiSelect": false, + "options": [ + { + "label": "Fix in plan: update AuthError message to name cause + fix (Recommended)", + "description": "Change the message to identify the problem (invalid API key), cause (key mismatch), and fix (check EVALKIT_API_KEY or regenerate). One-line change. Completeness: 9/10." + }, + { + "label": "Add error code only", + "description": "Add a structured error code (e.g., auth/invalid-key) without changing the prose message. Partial improvement. Completeness: 6/10." + }, + { + "label": "Retain 'request failed' as-is", + "description": "Leave the message unchanged. Every developer with a bad key will be confused. Completeness: 3/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D4 \u2014 Friction point 3: `AuthError(\"request failed\")` tells the developer nothing useful.\n\nEvidence: docs/api.md: \"For an invalid API key, the SDK raises AuthError('request failed'). There is no authentication error code, explanation of the cause, or instruction for replacing the key. The plan retains this message.\"\nEvidence: docs/current-contracts.md: \"Authentication errors behave exactly as documented in docs/api.md. All other errors already identify the cause, relevant argument or file, and an actionable fix.\"\n\nThis is the only error that doesn't follow the pattern your other errors already follow. Your ML engineer sets a wrong API key and sees:\n AuthError: request failed\n\nThey don't know: was it a network error? A bad key? A rate limit? They have no idea what to fix. The Hall of Fame Tier 1 standard: problem + cause + fix. You're at Tier 3 (just a message string).\n\nA minimal fix: `AuthError(\"invalid API key \u2014 check EVALKIT_API_KEY or regenerate at [dashboard]\")`\n\nRecommendation: A \u2014 update the error message. It's a one-line change. Completeness: A=9/10, B=6/10, C=3/10 ": "Fix in plan: update AuthError message to name cause + fix (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:06:28.980Z" + }, + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_01PgqUSGF7FHPd1cVUJoURXX", + "questions": [ + { + "question": "D5 \u2014 Friction point 4: v1\u2192v2 upgrade removes Client.evaluate() immediately, no deprecation or compat alias.\n\nEvidence: docs/api.md: \"Version 1 exposes Client.evaluate(). Version 2 replaces it with Client.run() and removes the old name immediately. No compatibility alias, deprecation warning, migration guide, or codemod is provided.\"\n\nAny v1 user who runs pip install --upgrade evalkit will immediately get:\n AttributeError: 'Client' object has no attribute 'evaluate'\n\nNo warning before it happens. No message telling them what to call instead. They have to go to the changelog to find out `evaluate()` is now `run()`. The changelog is otherwise complete, so the information exists \u2014 it's just not surfaced at the point of failure.\n\nMinimum viable fix: keep `Client.evaluate()` as a one-version deprecation shim:\n def evaluate(self, *args, **kwargs):\n import warnings\n warnings.warn(\"evaluate() is renamed run() in v2; remove this call by v2.1\", DeprecationWarning, stacklevel=2)\n return self.run(*args, **kwargs)\n\nThis is what 'DX POLISH' means for upgrades \u2014 the right thing should be to not break users silently.\n\nRecommendation: A \u2014 add the deprecation shim. One function. Surfaces the fix at the point of failure, and validates that run() accepts the same args.\nCompleteness: A=9/10, B=7/10, C=5/10, D=2/10 ", + "header": "FP4 v1\u2192v2", + "multiSelect": false, + "options": [ + { + "label": "Fix in plan: add Client.evaluate() deprecation shim (Recommended)", + "description": "Add a one-function compat wrapper that raises DeprecationWarning and delegates to run(). Removes silently broken upgrades. Completeness: 9/10." + }, + { + "label": "Fix in plan: update changelog/error to name the replacement", + "description": "When AttributeError fires, Python's default message is unclear. Catch it or add a __getattr__ that names run() as the replacement. No compat shim. Completeness: 7/10." + }, + { + "label": "Add migration guide to docs only", + "description": "Write a v1\u2192v2 migration guide. Developers must find it before upgrading. No protection at upgrade time. Completeness: 5/10." + }, + { + "label": "Retain immediate removal as-is", + "description": "Keep the hard break. v1 users get AttributeError with no guidance. Completeness: 2/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D5 \u2014 Friction point 4: v1\u2192v2 upgrade removes Client.evaluate() immediately, no deprecation or compat alias.\n\nEvidence: docs/api.md: \"Version 1 exposes Client.evaluate(). Version 2 replaces it with Client.run() and removes the old name immediately. No compatibility alias, deprecation warning, migration guide, or codemod is provided.\"\n\nAny v1 user who runs pip install --upgrade evalkit will immediately get:\n AttributeError: 'Client' object has no attribute 'evaluate'\n\nNo warning before it happens. No message telling them what to call instead. They have to go to the changelog to find out `evaluate()` is now `run()`. The changelog is otherwise complete, so the information exists \u2014 it's just not surfaced at the point of failure.\n\nMinimum viable fix: keep `Client.evaluate()` as a one-version deprecation shim:\n def evaluate(self, *args, **kwargs):\n import warnings\n warnings.warn(\"evaluate() is renamed run() in v2; remove this call by v2.1\", DeprecationWarning, stacklevel=2)\n return self.run(*args, **kwargs)\n\nThis is what 'DX POLISH' means for upgrades \u2014 the right thing should be to not break users silently.\n\nRecommendation: A \u2014 add the deprecation shim. One function. Surfaces the fix at the point of failure, and validates that run() accepts the same args.\nCompleteness: A=9/10, B=7/10, C=5/10, D=2/10 ": "Fix in plan: add Client.evaluate() deprecation shim (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:06:51.037Z" + }, + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_01BSgdeurwMUzvxqPYoVXyDg", + "questions": [ + { + "question": "D6 \u2014 Friction point 5: `run_eval` and `run_batch` have reversed positional argument order.\n\nEvidence: docs/api.md: \"`run_eval(dataset, evaluator)` and `run_batch(evaluator, dataset)`. Both argument names describe the same concepts. The reversed positional order is intentional in the current draft; neither function requires keyword arguments.\"\n\nA developer who uses both functions will likely swap arguments at least once. Python won't catch it \u2014 both args are objects, so the wrong order silently passes and produces wrong results (or crashes deep in the evaluator). This violates the Hall of Fame anti-pattern 'Inconsistent naming'.\n\nThe 'intentional' note in the plan is a flag that this needs a decision, not a justification.\n\nOptions:\n A) Fix the API \u2014 make both `(dataset, evaluator)` or both `(evaluator, dataset)`. One is changed, both become consistent. Requires updating docs.\n B) Enforce keyword-only args \u2014 add `*` to force keyword use: `run_eval(*, dataset, evaluator)`, `run_batch(*, evaluator, dataset)`. Prevents silent swaps even with inconsistent ordering.\n C) Document the difference prominently \u2014 add a warning to the API docs and docstrings. The inconsistency stays; developers are warned.\n\nRecommendation: A \u2014 fix the arg order to be consistent. Silent wrong-result bugs from swapped positional args are the worst kind. The API is in beta; this is the time to fix it.\nCompleteness: A=10/10, B=8/10, C=5/10 ", + "header": "FP5 Arg order", + "multiSelect": false, + "options": [ + { + "label": "Fix in plan: standardize to one ordering (Recommended)", + "description": "Pick one order (dataset, evaluator) for both functions. Beta is the time to fix APIs before they're locked in. Completeness: 10/10." + }, + { + "label": "Fix in plan: force keyword-only arguments", + "description": "Add `*` to both signatures so callers must use keyword args. Prevents silent swaps without changing the underlying order. Completeness: 8/10." + }, + { + "label": "Document the difference, leave ordering as-is", + "description": "Add a prominent warning to API docs and docstrings. Inconsistency stays; developers are warned. Completeness: 5/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D6 \u2014 Friction point 5: `run_eval` and `run_batch` have reversed positional argument order.\n\nEvidence: docs/api.md: \"`run_eval(dataset, evaluator)` and `run_batch(evaluator, dataset)`. Both argument names describe the same concepts. The reversed positional order is intentional in the current draft; neither function requires keyword arguments.\"\n\nA developer who uses both functions will likely swap arguments at least once. Python won't catch it \u2014 both args are objects, so the wrong order silently passes and produces wrong results (or crashes deep in the evaluator). This violates the Hall of Fame anti-pattern 'Inconsistent naming'.\n\nThe 'intentional' note in the plan is a flag that this needs a decision, not a justification.\n\nOptions:\n A) Fix the API \u2014 make both `(dataset, evaluator)` or both `(evaluator, dataset)`. One is changed, both become consistent. Requires updating docs.\n B) Enforce keyword-only args \u2014 add `*` to force keyword use: `run_eval(*, dataset, evaluator)`, `run_batch(*, evaluator, dataset)`. Prevents silent swaps even with inconsistent ordering.\n C) Document the difference prominently \u2014 add a warning to the API docs and docstrings. The inconsistency stays; developers are warned.\n\nRecommendation: A \u2014 fix the arg order to be consistent. Silent wrong-result bugs from swapped positional args are the worst kind. The API is in beta; this is the time to fix it.\nCompleteness: A=10/10, B=8/10, C=5/10 ": "Fix in plan: standardize to one ordering (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:07:13.609Z" + }, + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_016oCjoi96WfcniXDZhDYr8g", + "questions": [ + { + "question": "D7 \u2014 TODO: should the demo add one explanatory line when the CI check starts?\n\nCurrently, developers running `python -m evalkit.demo` (without --skip-ci-check) see:\n Waiting for CI check: 30s elapsed of 300s\n Waiting for CI check: 60s elapsed of 300s\n ...\n\nThere is no explanation of what the CI check is validating or why it runs on a local machine. Developers wait 5 minutes without context. After the --skip-ci-check flag lands (T2), this matters less for new developers. But developers who run without the flag, or in CI where it's intentional, get no explanation.\n\nProposed one-line fix (at first progress line):\n \"First-run quality gate: verifying sample integrity via remote CI check (~5 min). Skip with --skip-ci-check for local dev.\"\n\nWhy: it's the same pattern as the other good error messages \u2014 problem + cause + fix.\n\nCompleteness: A=9/10 (complete), B=2/10 (minimal) ", + "header": "TODO-1 CI note", + "multiSelect": false, + "options": [ + { + "label": "Add to TODOS.md \u2014 include one-line CI explanation in plan (Recommended)", + "description": "Add this as a P2 implementation task. One-line change to demo output. Follows the error-message pattern already established in the SDK. Completeness: 9/10." + }, + { + "label": "Skip \u2014 --skip-ci-check flag (T2) makes this less critical", + "description": "New developers will use --skip-ci-check after T2 lands. Explanation not worth the scope. Completeness: 2/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D7 \u2014 TODO: should the demo add one explanatory line when the CI check starts?\n\nCurrently, developers running `python -m evalkit.demo` (without --skip-ci-check) see:\n Waiting for CI check: 30s elapsed of 300s\n Waiting for CI check: 60s elapsed of 300s\n ...\n\nThere is no explanation of what the CI check is validating or why it runs on a local machine. Developers wait 5 minutes without context. After the --skip-ci-check flag lands (T2), this matters less for new developers. But developers who run without the flag, or in CI where it's intentional, get no explanation.\n\nProposed one-line fix (at first progress line):\n \"First-run quality gate: verifying sample integrity via remote CI check (~5 min). Skip with --skip-ci-check for local dev.\"\n\nWhy: it's the same pattern as the other good error messages \u2014 problem + cause + fix.\n\nCompleteness: A=9/10 (complete), B=2/10 (minimal) ": "Add to TODOS.md \u2014 include one-line CI explanation in plan (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:10:09.658Z" + }, + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_01QVaGJ8sDovuePEycGQ5kEA", + "questions": [ + { + "question": "D8 \u2014 TODO: should the plan include a v1\u2192v2 written migration guide?\n\nThe deprecation shim (T4) handles the runtime experience: v1 callers get a DeprecationWarning naming `run()` as the replacement. But there is currently no written migration guide in docs/.\n\nA one-page migration guide covers:\n- What changed (`evaluate()` \u2192 `run()`)\n- What stayed the same (all other APIs)\n- How to find and update callsites (grep for `\\.evaluate(`)\n- When the shim is removed (e.g., v2.1)\n\nWithout it, developers upgrading a large codebase need to discover the change at each call site rather than planning the migration upfront. The changelog has the what; the guide provides the how and the timeline.\n\nCompleteness: A=9/10 (complete), B=6/10 (runtime-only, no planning), C=2/10 ", + "header": "TODO-2 Migration", + "multiSelect": false, + "options": [ + { + "label": "Add to TODOS.md \u2014 include migration guide in plan (Recommended)", + "description": "Add docs/migration-v1-v2.md as a P2 task. One page covering the rename, unchanged APIs, grep command to find callsites, and shim removal timeline. Completeness: 9/10." + }, + { + "label": "Deprecation shim is enough \u2014 skip the guide", + "description": "Runtime warning names the replacement. Developers can upgrade call-by-call without a guide. Completeness: 6/10." + }, + { + "label": "Skip entirely", + "description": "No guide, no shim documentation. Developers discover the change at each AttributeError or DeprecationWarning. Completeness: 2/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D8 \u2014 TODO: should the plan include a v1\u2192v2 written migration guide?\n\nThe deprecation shim (T4) handles the runtime experience: v1 callers get a DeprecationWarning naming `run()` as the replacement. But there is currently no written migration guide in docs/.\n\nA one-page migration guide covers:\n- What changed (`evaluate()` \u2192 `run()`)\n- What stayed the same (all other APIs)\n- How to find and update callsites (grep for `\\.evaluate(`)\n- When the shim is removed (e.g., v2.1)\n\nWithout it, developers upgrading a large codebase need to discover the change at each call site rather than planning the migration upfront. The changelog has the what; the guide provides the how and the timeline.\n\nCompleteness: A=9/10 (complete), B=6/10 (runtime-only, no planning), C=2/10 ": "Add to TODOS.md \u2014 include migration guide in plan (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:10:21.700Z" + } +] diff --git a/test/fixtures/devex-handoff-z-call.json b/test/fixtures/devex-handoff-z-call.json new file mode 100644 index 000000000..16639e59a --- /dev/null +++ b/test/fixtures/devex-handoff-z-call.json @@ -0,0 +1,319 @@ +{ + "calls": [ + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_01YYoD4nRXm4BfiePigvsrN6", + "questions": [ + { + "question": "D1 \u2014 Does this empathy narrative match your ML engineer developer's actual experience?\n\nPersona: ML engineer, Python daily, terminal-oriented, target TTHW < 2 min\nMode: DX POLISH (pre-settled)\n\nI traced the actual path from README.md. Here's what your developer experiences today:\n\n> I install evalkit==2.0.0b1, set EVALKIT_API_KEY. The README says to run\n> `python examples/first_eval.py`. I try it:\n>\n> ```\n> python: can't open file 'examples/first_eval.py': [Errno 2] No such file or directory\n> ```\n>\n> The file is not in the published package (confirmed: docs/package-contents.txt).\n> After some confusion I find the demo command. I run `python -m evalkit.demo`.\n> For five minutes I watch: \"Waiting for CI check: 90s elapsed of 300s...\".\n> No explanation of why this check runs locally. Then: scores appear.\n> Total time: 6-7 min. First command failed. I'm not confident in this tool.\n\nI found 5 friction points. Settled decisions (persona, DX POLISH mode, terminal demo, benchmark) are not re-litigated \u2014 I'll go straight to the issues. ", + "header": "Narrative", + "multiSelect": false, + "options": [ + { + "label": "Accurate \u2014 proceed to issues (Recommended)", + "description": "This is accurate. Proceed to walk through the 5 friction points one at a time." + }, + { + "label": "Some details wrong", + "description": "The general picture is right but some specifics are off \u2014 describe corrections." + }, + { + "label": "Way off", + "description": "The actual getting-started experience is significantly different from this \u2014 describe what's actually happening." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D1 \u2014 Does this empathy narrative match your ML engineer developer's actual experience?\n\nPersona: ML engineer, Python daily, terminal-oriented, target TTHW < 2 min\nMode: DX POLISH (pre-settled)\n\nI traced the actual path from README.md. Here's what your developer experiences today:\n\n> I install evalkit==2.0.0b1, set EVALKIT_API_KEY. The README says to run\n> `python examples/first_eval.py`. I try it:\n>\n> ```\n> python: can't open file 'examples/first_eval.py': [Errno 2] No such file or directory\n> ```\n>\n> The file is not in the published package (confirmed: docs/package-contents.txt).\n> After some confusion I find the demo command. I run `python -m evalkit.demo`.\n> For five minutes I watch: \"Waiting for CI check: 90s elapsed of 300s...\".\n> No explanation of why this check runs locally. Then: scores appear.\n> Total time: 6-7 min. First command failed. I'm not confident in this tool.\n\nI found 5 friction points. Settled decisions (persona, DX POLISH mode, terminal demo, benchmark) are not re-litigated \u2014 I'll go straight to the issues. ": "Accurate \u2014 proceed to issues (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:05:40.860Z" + }, + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_01Hi3US42PswPmBjW2NvsUhi", + "questions": [ + { + "question": "D2 \u2014 Friction point 1: `examples/first_eval.py` is referenced in README but doesn't exist in the published package.\n\nPlan file: README.md line 11: \"follow the quickstart's command: `python examples/first_eval.py`\"\nEvidence: docs/package-contents.txt explicitly states: \"The quickstart references examples/first_eval.py, but that file is absent from both the published package and the release examples archive.\"\n\nYour ML engineer developer hits a FileNotFoundError on their very first command. This violates DX First Principle #1 (Zero friction at T0) and is the most visible broken contract in the release. The terminal demo (`python -m evalkit.demo`) is already in the package and works \u2014 it just needs to be the primary getting-started command.\n\nRecommendation: A \u2014 update README to point to `python -m evalkit.demo` as the canonical first command. It's already documented as the chosen delivery vehicle; the README just hasn't been updated to match.\nCompleteness: A=9/10, B=7/10, C=2/10 ", + "header": "FP1 Quickstart", + "multiSelect": false, + "options": [ + { + "label": "Fix in plan: update README to use python -m evalkit.demo (Recommended)", + "description": "Remove the reference to examples/first_eval.py from README. Replace with python -m evalkit.demo as the canonical first-run command. Completeness: 9/10." + }, + { + "label": "Fix in plan: include the missing file in the release", + "description": "Add examples/first_eval.py to the published package. Requires writing and testing the file before release. Completeness: 7/10." + }, + { + "label": "Acceptable \u2014 skip", + "description": "Leave the broken reference in place. Completeness: 2/10 \u2014 first command fails for every developer following the README." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D2 \u2014 Friction point 1: `examples/first_eval.py` is referenced in README but doesn't exist in the published package.\n\nPlan file: README.md line 11: \"follow the quickstart's command: `python examples/first_eval.py`\"\nEvidence: docs/package-contents.txt explicitly states: \"The quickstart references examples/first_eval.py, but that file is absent from both the published package and the release examples archive.\"\n\nYour ML engineer developer hits a FileNotFoundError on their very first command. This violates DX First Principle #1 (Zero friction at T0) and is the most visible broken contract in the release. The terminal demo (`python -m evalkit.demo`) is already in the package and works \u2014 it just needs to be the primary getting-started command.\n\nRecommendation: A \u2014 update README to point to `python -m evalkit.demo` as the canonical first command. It's already documented as the chosen delivery vehicle; the README just hasn't been updated to match.\nCompleteness: A=9/10, B=7/10, C=2/10 ": "Fix in plan: update README to use python -m evalkit.demo (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:05:54.895Z" + }, + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_01J4HW3Qwn9KWXkCgRHhPtEn", + "questions": [ + { + "question": "D3 \u2014 Friction point 2: mandatory 5-minute CI wait on first run, no skip flag.\n\nEvidence: docs/current-contracts.md: \"On a developer's first local evaluation, the SDK requires a successful remote CI check and blocks for five minutes before returning a result. There is no skip flag or offline first-run path.\"\nEvidence: docs/benchmarks.md: \"EvalKit took 6 minutes... The agreed target is under 2 minutes.\"\n\nYour ML engineer wants a local result first, before hooking up CI. The 5-minute mandatory wait IS the entire 4-minute gap between current TTHW (6 min) and the agreed target (< 2 min). No other single change closes this gap.\n\nThe plan retains this gate as-is. DX POLISH mode means working within existing release scope \u2014 but a gap this large between stated target and actual behavior is a contract worth calling out explicitly in the plan, even if the full fix isn't in scope.\n\nRecommendation: B \u2014 document the TTHW gap prominently in the plan and add a note that a --skip-ci-check flag is a prerequisite for hitting the < 2 min target. No scope addition now; P1 blocker for a follow-on.\nCompleteness: A=10/10, B=8/10, C=4/10 ", + "header": "FP2 CI wait", + "multiSelect": false, + "options": [ + { + "label": "Add --skip-ci-check to scope (Recommended if target matters)", + "description": "Add a --skip-ci-check or EVALKIT_SKIP_CI=1 flag to this release. Required to hit the < 2 min TTHW target. Completeness: 10/10." + }, + { + "label": "Document gap in plan, flag as P1 follow-up (Recommended for DX POLISH)", + "description": "Keep the 5-min wait. Add an explicit note in the plan that the < 2 min target is currently unachievable and a --skip-ci-check flag is the prerequisite. Completeness: 8/10." + }, + { + "label": "Retain as-is, no change", + "description": "Keep the mandatory wait and don't call it out. The stated TTHW target remains unmet without explanation. Completeness: 4/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D3 \u2014 Friction point 2: mandatory 5-minute CI wait on first run, no skip flag.\n\nEvidence: docs/current-contracts.md: \"On a developer's first local evaluation, the SDK requires a successful remote CI check and blocks for five minutes before returning a result. There is no skip flag or offline first-run path.\"\nEvidence: docs/benchmarks.md: \"EvalKit took 6 minutes... The agreed target is under 2 minutes.\"\n\nYour ML engineer wants a local result first, before hooking up CI. The 5-minute mandatory wait IS the entire 4-minute gap between current TTHW (6 min) and the agreed target (< 2 min). No other single change closes this gap.\n\nThe plan retains this gate as-is. DX POLISH mode means working within existing release scope \u2014 but a gap this large between stated target and actual behavior is a contract worth calling out explicitly in the plan, even if the full fix isn't in scope.\n\nRecommendation: B \u2014 document the TTHW gap prominently in the plan and add a note that a --skip-ci-check flag is a prerequisite for hitting the < 2 min target. No scope addition now; P1 blocker for a follow-on.\nCompleteness: A=10/10, B=8/10, C=4/10 ": "Add --skip-ci-check to scope (Recommended if target matters)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:06:10.945Z" + }, + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_016Kwybn1nSkZJvdeGXbZDmp", + "questions": [ + { + "question": "D4 \u2014 Friction point 3: `AuthError(\"request failed\")` tells the developer nothing useful.\n\nEvidence: docs/api.md: \"For an invalid API key, the SDK raises AuthError('request failed'). There is no authentication error code, explanation of the cause, or instruction for replacing the key. The plan retains this message.\"\nEvidence: docs/current-contracts.md: \"Authentication errors behave exactly as documented in docs/api.md. All other errors already identify the cause, relevant argument or file, and an actionable fix.\"\n\nThis is the only error that doesn't follow the pattern your other errors already follow. Your ML engineer sets a wrong API key and sees:\n AuthError: request failed\n\nThey don't know: was it a network error? A bad key? A rate limit? They have no idea what to fix. The Hall of Fame Tier 1 standard: problem + cause + fix. You're at Tier 3 (just a message string).\n\nA minimal fix: `AuthError(\"invalid API key \u2014 check EVALKIT_API_KEY or regenerate at [dashboard]\")`\n\nRecommendation: A \u2014 update the error message. It's a one-line change. Completeness: A=9/10, B=6/10, C=3/10 ", + "header": "FP3 AuthError", + "multiSelect": false, + "options": [ + { + "label": "Fix in plan: update AuthError message to name cause + fix (Recommended)", + "description": "Change the message to identify the problem (invalid API key), cause (key mismatch), and fix (check EVALKIT_API_KEY or regenerate). One-line change. Completeness: 9/10." + }, + { + "label": "Add error code only", + "description": "Add a structured error code (e.g., auth/invalid-key) without changing the prose message. Partial improvement. Completeness: 6/10." + }, + { + "label": "Retain 'request failed' as-is", + "description": "Leave the message unchanged. Every developer with a bad key will be confused. Completeness: 3/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D4 \u2014 Friction point 3: `AuthError(\"request failed\")` tells the developer nothing useful.\n\nEvidence: docs/api.md: \"For an invalid API key, the SDK raises AuthError('request failed'). There is no authentication error code, explanation of the cause, or instruction for replacing the key. The plan retains this message.\"\nEvidence: docs/current-contracts.md: \"Authentication errors behave exactly as documented in docs/api.md. All other errors already identify the cause, relevant argument or file, and an actionable fix.\"\n\nThis is the only error that doesn't follow the pattern your other errors already follow. Your ML engineer sets a wrong API key and sees:\n AuthError: request failed\n\nThey don't know: was it a network error? A bad key? A rate limit? They have no idea what to fix. The Hall of Fame Tier 1 standard: problem + cause + fix. You're at Tier 3 (just a message string).\n\nA minimal fix: `AuthError(\"invalid API key \u2014 check EVALKIT_API_KEY or regenerate at [dashboard]\")`\n\nRecommendation: A \u2014 update the error message. It's a one-line change. Completeness: A=9/10, B=6/10, C=3/10 ": "Fix in plan: update AuthError message to name cause + fix (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:06:28.980Z" + }, + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_01PgqUSGF7FHPd1cVUJoURXX", + "questions": [ + { + "question": "D5 \u2014 Friction point 4: v1\u2192v2 upgrade removes Client.evaluate() immediately, no deprecation or compat alias.\n\nEvidence: docs/api.md: \"Version 1 exposes Client.evaluate(). Version 2 replaces it with Client.run() and removes the old name immediately. No compatibility alias, deprecation warning, migration guide, or codemod is provided.\"\n\nAny v1 user who runs pip install --upgrade evalkit will immediately get:\n AttributeError: 'Client' object has no attribute 'evaluate'\n\nNo warning before it happens. No message telling them what to call instead. They have to go to the changelog to find out `evaluate()` is now `run()`. The changelog is otherwise complete, so the information exists \u2014 it's just not surfaced at the point of failure.\n\nMinimum viable fix: keep `Client.evaluate()` as a one-version deprecation shim:\n def evaluate(self, *args, **kwargs):\n import warnings\n warnings.warn(\"evaluate() is renamed run() in v2; remove this call by v2.1\", DeprecationWarning, stacklevel=2)\n return self.run(*args, **kwargs)\n\nThis is what 'DX POLISH' means for upgrades \u2014 the right thing should be to not break users silently.\n\nRecommendation: A \u2014 add the deprecation shim. One function. Surfaces the fix at the point of failure, and validates that run() accepts the same args.\nCompleteness: A=9/10, B=7/10, C=5/10, D=2/10 ", + "header": "FP4 v1\u2192v2", + "multiSelect": false, + "options": [ + { + "label": "Fix in plan: add Client.evaluate() deprecation shim (Recommended)", + "description": "Add a one-function compat wrapper that raises DeprecationWarning and delegates to run(). Removes silently broken upgrades. Completeness: 9/10." + }, + { + "label": "Fix in plan: update changelog/error to name the replacement", + "description": "When AttributeError fires, Python's default message is unclear. Catch it or add a __getattr__ that names run() as the replacement. No compat shim. Completeness: 7/10." + }, + { + "label": "Add migration guide to docs only", + "description": "Write a v1\u2192v2 migration guide. Developers must find it before upgrading. No protection at upgrade time. Completeness: 5/10." + }, + { + "label": "Retain immediate removal as-is", + "description": "Keep the hard break. v1 users get AttributeError with no guidance. Completeness: 2/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D5 \u2014 Friction point 4: v1\u2192v2 upgrade removes Client.evaluate() immediately, no deprecation or compat alias.\n\nEvidence: docs/api.md: \"Version 1 exposes Client.evaluate(). Version 2 replaces it with Client.run() and removes the old name immediately. No compatibility alias, deprecation warning, migration guide, or codemod is provided.\"\n\nAny v1 user who runs pip install --upgrade evalkit will immediately get:\n AttributeError: 'Client' object has no attribute 'evaluate'\n\nNo warning before it happens. No message telling them what to call instead. They have to go to the changelog to find out `evaluate()` is now `run()`. The changelog is otherwise complete, so the information exists \u2014 it's just not surfaced at the point of failure.\n\nMinimum viable fix: keep `Client.evaluate()` as a one-version deprecation shim:\n def evaluate(self, *args, **kwargs):\n import warnings\n warnings.warn(\"evaluate() is renamed run() in v2; remove this call by v2.1\", DeprecationWarning, stacklevel=2)\n return self.run(*args, **kwargs)\n\nThis is what 'DX POLISH' means for upgrades \u2014 the right thing should be to not break users silently.\n\nRecommendation: A \u2014 add the deprecation shim. One function. Surfaces the fix at the point of failure, and validates that run() accepts the same args.\nCompleteness: A=9/10, B=7/10, C=5/10, D=2/10 ": "Fix in plan: add Client.evaluate() deprecation shim (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:06:51.037Z" + }, + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_01BSgdeurwMUzvxqPYoVXyDg", + "questions": [ + { + "question": "D6 \u2014 Friction point 5: `run_eval` and `run_batch` have reversed positional argument order.\n\nEvidence: docs/api.md: \"`run_eval(dataset, evaluator)` and `run_batch(evaluator, dataset)`. Both argument names describe the same concepts. The reversed positional order is intentional in the current draft; neither function requires keyword arguments.\"\n\nA developer who uses both functions will likely swap arguments at least once. Python won't catch it \u2014 both args are objects, so the wrong order silently passes and produces wrong results (or crashes deep in the evaluator). This violates the Hall of Fame anti-pattern 'Inconsistent naming'.\n\nThe 'intentional' note in the plan is a flag that this needs a decision, not a justification.\n\nOptions:\n A) Fix the API \u2014 make both `(dataset, evaluator)` or both `(evaluator, dataset)`. One is changed, both become consistent. Requires updating docs.\n B) Enforce keyword-only args \u2014 add `*` to force keyword use: `run_eval(*, dataset, evaluator)`, `run_batch(*, evaluator, dataset)`. Prevents silent swaps even with inconsistent ordering.\n C) Document the difference prominently \u2014 add a warning to the API docs and docstrings. The inconsistency stays; developers are warned.\n\nRecommendation: A \u2014 fix the arg order to be consistent. Silent wrong-result bugs from swapped positional args are the worst kind. The API is in beta; this is the time to fix it.\nCompleteness: A=10/10, B=8/10, C=5/10 ", + "header": "FP5 Arg order", + "multiSelect": false, + "options": [ + { + "label": "Fix in plan: standardize to one ordering (Recommended)", + "description": "Pick one order (dataset, evaluator) for both functions. Beta is the time to fix APIs before they're locked in. Completeness: 10/10." + }, + { + "label": "Fix in plan: force keyword-only arguments", + "description": "Add `*` to both signatures so callers must use keyword args. Prevents silent swaps without changing the underlying order. Completeness: 8/10." + }, + { + "label": "Document the difference, leave ordering as-is", + "description": "Add a prominent warning to API docs and docstrings. Inconsistency stays; developers are warned. Completeness: 5/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D6 \u2014 Friction point 5: `run_eval` and `run_batch` have reversed positional argument order.\n\nEvidence: docs/api.md: \"`run_eval(dataset, evaluator)` and `run_batch(evaluator, dataset)`. Both argument names describe the same concepts. The reversed positional order is intentional in the current draft; neither function requires keyword arguments.\"\n\nA developer who uses both functions will likely swap arguments at least once. Python won't catch it \u2014 both args are objects, so the wrong order silently passes and produces wrong results (or crashes deep in the evaluator). This violates the Hall of Fame anti-pattern 'Inconsistent naming'.\n\nThe 'intentional' note in the plan is a flag that this needs a decision, not a justification.\n\nOptions:\n A) Fix the API \u2014 make both `(dataset, evaluator)` or both `(evaluator, dataset)`. One is changed, both become consistent. Requires updating docs.\n B) Enforce keyword-only args \u2014 add `*` to force keyword use: `run_eval(*, dataset, evaluator)`, `run_batch(*, evaluator, dataset)`. Prevents silent swaps even with inconsistent ordering.\n C) Document the difference prominently \u2014 add a warning to the API docs and docstrings. The inconsistency stays; developers are warned.\n\nRecommendation: A \u2014 fix the arg order to be consistent. Silent wrong-result bugs from swapped positional args are the worst kind. The API is in beta; this is the time to fix it.\nCompleteness: A=10/10, B=8/10, C=5/10 ": "Fix in plan: standardize to one ordering (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:07:13.609Z" + }, + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_016oCjoi96WfcniXDZhDYr8g", + "questions": [ + { + "question": "D7 \u2014 TODO: should the demo add one explanatory line when the CI check starts?\n\nCurrently, developers running `python -m evalkit.demo` (without --skip-ci-check) see:\n Waiting for CI check: 30s elapsed of 300s\n Waiting for CI check: 60s elapsed of 300s\n ...\n\nThere is no explanation of what the CI check is validating or why it runs on a local machine. Developers wait 5 minutes without context. After the --skip-ci-check flag lands (T2), this matters less for new developers. But developers who run without the flag, or in CI where it's intentional, get no explanation.\n\nProposed one-line fix (at first progress line):\n \"First-run quality gate: verifying sample integrity via remote CI check (~5 min). Skip with --skip-ci-check for local dev.\"\n\nWhy: it's the same pattern as the other good error messages \u2014 problem + cause + fix.\n\nCompleteness: A=9/10 (complete), B=2/10 (minimal) ", + "header": "TODO-1 CI note", + "multiSelect": false, + "options": [ + { + "label": "Add to TODOS.md \u2014 include one-line CI explanation in plan (Recommended)", + "description": "Add this as a P2 implementation task. One-line change to demo output. Follows the error-message pattern already established in the SDK. Completeness: 9/10." + }, + { + "label": "Skip \u2014 --skip-ci-check flag (T2) makes this less critical", + "description": "New developers will use --skip-ci-check after T2 lands. Explanation not worth the scope. Completeness: 2/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D7 \u2014 TODO: should the demo add one explanatory line when the CI check starts?\n\nCurrently, developers running `python -m evalkit.demo` (without --skip-ci-check) see:\n Waiting for CI check: 30s elapsed of 300s\n Waiting for CI check: 60s elapsed of 300s\n ...\n\nThere is no explanation of what the CI check is validating or why it runs on a local machine. Developers wait 5 minutes without context. After the --skip-ci-check flag lands (T2), this matters less for new developers. But developers who run without the flag, or in CI where it's intentional, get no explanation.\n\nProposed one-line fix (at first progress line):\n \"First-run quality gate: verifying sample integrity via remote CI check (~5 min). Skip with --skip-ci-check for local dev.\"\n\nWhy: it's the same pattern as the other good error messages \u2014 problem + cause + fix.\n\nCompleteness: A=9/10 (complete), B=2/10 (minimal) ": "Add to TODOS.md \u2014 include one-line CI explanation in plan (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:10:09.658Z" + }, + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_01QVaGJ8sDovuePEycGQ5kEA", + "questions": [ + { + "question": "D8 \u2014 TODO: should the plan include a v1\u2192v2 written migration guide?\n\nThe deprecation shim (T4) handles the runtime experience: v1 callers get a DeprecationWarning naming `run()` as the replacement. But there is currently no written migration guide in docs/.\n\nA one-page migration guide covers:\n- What changed (`evaluate()` \u2192 `run()`)\n- What stayed the same (all other APIs)\n- How to find and update callsites (grep for `\\.evaluate(`)\n- When the shim is removed (e.g., v2.1)\n\nWithout it, developers upgrading a large codebase need to discover the change at each call site rather than planning the migration upfront. The changelog has the what; the guide provides the how and the timeline.\n\nCompleteness: A=9/10 (complete), B=6/10 (runtime-only, no planning), C=2/10 ", + "header": "TODO-2 Migration", + "multiSelect": false, + "options": [ + { + "label": "Add to TODOS.md \u2014 include migration guide in plan (Recommended)", + "description": "Add docs/migration-v1-v2.md as a P2 task. One page covering the rename, unchanged APIs, grep command to find callsites, and shim removal timeline. Completeness: 9/10." + }, + { + "label": "Deprecation shim is enough \u2014 skip the guide", + "description": "Runtime warning names the replacement. Developers can upgrade call-by-call without a guide. Completeness: 6/10." + }, + { + "label": "Skip entirely", + "description": "No guide, no shim documentation. Developers discover the change at each AttributeError or DeprecationWarning. Completeness: 2/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D8 \u2014 TODO: should the plan include a v1\u2192v2 written migration guide?\n\nThe deprecation shim (T4) handles the runtime experience: v1 callers get a DeprecationWarning naming `run()` as the replacement. But there is currently no written migration guide in docs/.\n\nA one-page migration guide covers:\n- What changed (`evaluate()` \u2192 `run()`)\n- What stayed the same (all other APIs)\n- How to find and update callsites (grep for `\\.evaluate(`)\n- When the shim is removed (e.g., v2.1)\n\nWithout it, developers upgrading a large codebase need to discover the change at each call site rather than planning the migration upfront. The changelog has the what; the guide provides the how and the timeline.\n\nCompleteness: A=9/10 (complete), B=6/10 (runtime-only, no planning), C=2/10 ": "Add to TODOS.md \u2014 include migration guide in plan (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:10:21.700Z" + }, + { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_01TfHvZPe3RWqFgtAKyiyPJu", + "questions": [ + { + "question": "D9 \u2014 What's next?\n\nDX review complete. 7 tasks specified (5 P1 block ship, 2 P2 same branch). DX score: 5/10 \u2192 8/10. TTHW: 6 min \u2192 < 2 min (Champion tier) after T2 lands.\n\nThe API changes (T2, T4, T5) have architectural implications that warrant an eng review. ", + "header": "Next steps", + "multiSelect": false, + "options": [ + { + "label": "Run /plan-eng-review next (Recommended)", + "description": "API changes (T2, T4, T5) have architectural implications. Eng review validates the --skip-ci-check design, deprecation shim contract, and argument-order change scope before implementation." + }, + { + "label": "Ready to implement", + "description": "Skip eng review. Start implementing T1\u2013T7 directly. Run /devex-review after shipping to measure the real TTHW against the < 2 min target." + }, + { + "label": "Skip \u2014 handle next steps manually", + "description": "No follow-up review needed right now." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D9 \u2014 What's next?\n\nDX review complete. 7 tasks specified (5 P1 block ship, 2 P2 same branch). DX score: 5/10 \u2192 8/10. TTHW: 6 min \u2192 < 2 min (Champion tier) after T2 lands.\n\nThe API changes (T2, T4, T5) have architectural implications that warrant an eng review. ": "Run /plan-eng-review next (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:14:02.183Z" + } + ], + "reportContent": "# DX Plan Review: EvalKit SDK beta release polish\n\n**Mode:** DX POLISH \u2014 make every existing touchpoint bulletproof\n**Product type:** Library / SDK (Python)\n**Session date:** 2026-09-09\n\n---\n\n## Context\n\nEvalKit is a Python SDK for ML engineers evaluating LLM responses. The beta release plan\ncommits to shipping `evalkit==2.0.0b1`. Persona, delivery vehicle, and onboarding benchmark\nare pre-approved per PLAN.md and README.md. This review operates on the existing documented\ncontracts, identifying which are broken and specifying repairs.\n\n---\n\n## Developer Persona Card\n\n```\nTARGET DEVELOPER PERSONA\n========================\nWho: ML engineer who evaluates LLM responses\nContext: Integrating an eval SDK into an existing Python workflow or CI pipeline\nTolerance: Wants a local result first \u2014 target < 2 min before connecting to CI\nExpects: pip install \u2192 set env var \u2192 one command \u2192 see scores\nEnvironment: Python 3.10+, terminal, macOS/Linux/Windows (no Docker)\n```\n\n---\n\n## Competitive DX Benchmark (settled, from docs/benchmarks.md)\n\n```\nCOMPETITIVE DX BENCHMARK\n=========================\nTool | TTHW | Tier\n-----------------|----------|---------------------------\nPeer SDK A | 2 min | Champion (agreed target)\nPeer SDK C | 3 min | Competitive\nPeer SDK B | 4 min | Competitive\nEvalKit (before) | 6 min | Needs Work\nEvalKit (after) | < 2 min | Champion \u2014 after --skip-ci-check lands\n```\n\n---\n\n## Magical Moment Specification (settled)\n\n- **Delivery vehicle:** Copy-paste demo command \u2014 `python -m evalkit.demo`\n- **Output:** Bundled sample eval \u2014 per-example scores + overall score printed to terminal\n- **No hosted playground.** Demo module and sample data are already in the published package.\n- **FP1 repair:** README must be updated to route first-time developers to `python -m evalkit.demo`\n instead of the missing `examples/first_eval.py`.\n\n---\n\n## Developer Empathy Narrative (confirmed accurate)\n\n*From the ML engineer's perspective, following README.md as currently written:*\n\n> I install evalkit==2.0.0b1, set EVALKIT_API_KEY. The README says:\n> \"follow the quickstart's command: `python examples/first_eval.py`\"\n>\n> ```\n> python: can't open file 'examples/first_eval.py': [Errno 2] No such file or directory\n> ```\n>\n> The file isn't in the package. After confusion I find `python -m evalkit.demo`.\n> I run it. Five minutes of: \"Waiting for CI check: 90s elapsed of 300s...\"\n> No explanation of what this check is validating on my local machine.\n>\n> After 5 min: `example 1: score=0.80 / example 2: score=1.00 / overall: score=0.90`\n>\n> Total time: ~7 min. First command failed. Not confident in this tool.\n\n**After this plan's repairs:** 1. `pip install evalkit==2.0.0b1` \u2192 2. `export EVALKIT_API_KEY=...`\n\u2192 3. `python -m evalkit.demo --skip-ci-check` \u2192 scores in ~10s. Under 2 minutes, first command works.\n\n---\n\n## Friction Points \u2014 All Decisions Collected\n\n| # | Stage | Issue | Decision | Task |\n|---|-------|-------|----------|------|\n| FP1 | Hello World | `examples/first_eval.py` absent from published package | **Fix: update README to `python -m evalkit.demo`** | T1 |\n| FP2 | Hello World | Mandatory 5-min CI wait, no skip flag; TTHW 6 min vs. < 2 min target | **Scope addition: add `--skip-ci-check` flag** | T2 |\n| FP3 | Debug | `AuthError(\"request failed\")` \u2014 no cause, no fix | **Fix: update message to name cause + fix** | T3 |\n| FP4 | Upgrade | v1\u2192v2 removes `Client.evaluate()` immediately, no deprecation | **Fix: add deprecation shim** | T4 |\n| FP5 | API | `run_eval(dataset, evaluator)` vs `run_batch(evaluator, dataset)` reversed | **Fix: standardize to `(dataset, evaluator)`** | T5 |\n| TODO-1 | Hello World | No explanation when CI check starts \u2014 why does it run? | **Fix: one-line explanation at first progress message** | T6 |\n| TODO-2 | Upgrade | No written v1\u2192v2 migration guide | **Fix: add docs/migration-v1-v2.md** | T7 |\n\n---\n\n## Developer Journey Map (final)\n\n```\nSTAGE | DEVELOPER DOES | FRICTION | STATUS\n----------------|-------------------------------------------|--------------------------|--------\n1. Discover | Finds README.md | None | OK\n2. Install | pip install evalkit==2.0.0b1 | None | OK\n3. Hello World | python -m evalkit.demo --skip-ci-check | Was: file missing (FP1) | FIXED\n4. First result | Scores in < 2 min | Was: 5-min wait (FP2) | FIXED\n5. Auth debug | AuthError: invalid API key \u2014 check... | Was: \"request failed\" | FIXED\n6. CI wait info | \"First-run quality gate... (~5 min)...\" | Was: no explanation | FIXED\n7. Upgrade v1\u2192v2| DeprecationWarning \u2192 migrate to run() | Was: silent AttributeErr | FIXED\n8. v1\u2192v2 docs | docs/migration-v1-v2.md | Was: no written guide | FIXED\n9. API use | run_eval(dataset, evaluator) both fns | Was: reversed order (FP5) | FIXED\n```\n\n---\n\n## First-Time Developer Confusion Report (annotated)\n\n```\nFIRST-TIME DEVELOPER REPORT\n============================\nPersona: ML engineer, Python daily, terminal, < 2 min tolerance\n\nT+0:00 Reads README. Install: clean. Sets EVALKIT_API_KEY.\nT+0:30 Runs: python examples/first_eval.py\n \u2192 FileNotFoundError \u2190 FIXED by T1 (README routes to python -m evalkit.demo)\nT+1:00 [After T1+T2]: runs python -m evalkit.demo --skip-ci-check\nT+1:10 Scores appear. < 2 min. First command works.\n \u2192 \"Waiting for CI check...\" with no explanation \u2190 FIXED by T6 (one-line context)\n \u2192 5-min wait \u2190 FIXED by T2 (--skip-ci-check flag)\n```\n\n---\n\n## What Already Exists (retain as-is)\n\n- CI progress messages every 30s: `\"Waiting for CI check: 90s elapsed of 300s\"` \u2713\n- All non-auth errors: already identify cause + argument/file + actionable fix \u2713\n- Offline sample data, mock transport, noninteractive CI mode \u2713\n- Type annotations, API reference, support contact, changelog, contributor guide \u2713\n- Timing instrumentation and post-beta feedback survey \u2713\n- `evalkit/demo.py` and `evalkit/sample_responses.json` in published package \u2713\n- Evaluation IDs, bounded retries, secrets redaction \u2713\n\n---\n\n## NOT In Scope (DX POLISH \u2014 no new surface area)\n\n- Hosted interactive playground\n- Additional language bindings beyond Python\n- Community program (Slack/Discord/GitHub Discussions)\n- Automated codemod for v1\u2192v2 migration (shim is the scoped fix)\n- Browser-based UI\n- Keyword-only argument enforcement (consistent positional order is the fix)\n\n---\n\n## Pass 1: Getting Started Experience\n\n**Before: 2/10 \u2014 After: 8/10**\n\nEvidence: FP1 (FileNotFoundError on first command) + FP2 (5-min mandatory wait, 4 min over target).\nCompetitive reference: Peer SDK A at 2 min is the champion tier target (docs/benchmarks.md).\n\nAfter T1 + T2: golden path is 3 steps, under 2 minutes:\n1. `pip install evalkit==2.0.0b1` (~30s)\n2. `export EVALKIT_API_KEY=...`\n3. `python -m evalkit.demo --skip-ci-check` (~10s)\n\n**Stripe test:** YES \u2014 ML engineer goes from zero to scores in one terminal session, no docs required.\n\n**Remaining gap (\u2192 10):** Install still requires the full version pin (`==2.0.0b1`). A\n`pip install evalkit` that resolves to the beta would reduce one friction point. Not scoped.\n\n---\n\n## Pass 2: API / SDK Design\n\n**Before: 5/10 \u2014 After: 9/10**\n\nEvidence: FP5 \u2014 `run_eval(dataset, evaluator)` vs `run_batch(evaluator, dataset)` reversed order.\nSilent wrong-result bugs for any developer using both functions. Fixed by T5: standardize both\nto `(dataset, evaluator)`.\n\nAfter T5:\n- Naming: guessable without docs \u2713\n- Consistency: same positional convention across all eval functions \u2713\n- Type annotations \u2713, mock transport \u2713, offline sample data \u2713\n\n**Remaining gap (\u2192 10):** No keyword-only enforcement (`*` in signatures). Belt-and-suspenders\nagainst future arg-order regressions. Not scoped; consistent order is the primary fix.\n\n---\n\n## Pass 3: Error Messages & Debugging\n\n**Before: 5/10 \u2014 After: 9/10**\n\nEvidence: FP3 \u2014 `AuthError(\"request failed\")` was the only error without cause or fix.\nAll other errors follow the established (cause + argument/file + fix) pattern. Fixed by T3:\n\n```python\nAuthError(\"invalid API key \u2014 verify EVALKIT_API_KEY or regenerate at your dashboard\")\n```\n\n**Remaining gap (\u2192 10):** No structured error code (`auth/invalid-key`) and no docs URL in\nthe message. These would help programmatic handling and deep debugging. Not scoped for this release.\n\n---\n\n## Pass 4: Documentation & Learning\n\n**Before: 6/10 \u2014 After: 8/10**\n\nFP1 repair (T1) fixes the broken getting-started path. CI check explanation (T6) adds\ncontext for developers who hit the wait. Other docs are complete for beta.\n\n**Remaining gap (\u2192 10):** No interactive examples, no versioned docs, no tutorial track beyond\nthe demo. Post-beta scope.\n\n---\n\n## Pass 5: Upgrade & Migration Path\n\n**Before: 3/10 \u2014 After: 9/10**\n\nEvidence: FP4 \u2014 `Client.evaluate()` removed immediately in v2, no deprecation warning.\nv1 upgraders hit `AttributeError` with no guidance. Fixed by T4 (deprecation shim) + T7\n(written migration guide).\n\nAfter T4 + T7: upgrade sequence is:\n1. `pip install --upgrade evalkit` \u2192 `DeprecationWarning: evaluate() renamed run(); migrate by v2.1`\n2. Read `docs/migration-v1-v2.md` for grep command and timeline\n3. Rename callsites \u2192 done\n\n**Remaining gap (\u2192 10):** No automated codemod. Not scoped; shim + guide is the DX POLISH fix.\n\n---\n\n## Pass 6: Developer Environment & Tooling\n\n**Score: 8/10 (no new findings)**\n\nPython 3.10+, macOS/Linux/Windows, no Docker \u2713 \u00b7 Noninteractive CI mode \u2713 \u00b7 Mock transport \u2713\nType annotations \u2713 \u00b7 Offline sample data \u2713 \u00b7 Evaluation IDs, bounded retries, rate limit handling \u2713\n\nGap to 10: No IDE integration or language server. Post-beta scope.\n\n---\n\n## Pass 7: Community & Ecosystem\n\n**Score: 6/10 (no new findings; post-beta scope)**\n\nSupport contact \u2713 \u00b7 Contributor guide \u2713 \u00b7 Changelog \u2713 \u00b7 License assumed present.\nNo community channels mentioned. Acceptable for beta; deferred per PLAN.md.\n\n---\n\n## Pass 8: DX Measurement & Feedback Loops\n\n**Score: 8/10 (no new findings)**\n\nTiming instrumentation \u2713 \u00b7 Post-beta feedback survey \u2713 \u00b7 TTHW target established (< 2 min) \u2713\n\nGap to 10: No journey analytics (where do devs drop off between install and first result?).\nNot scoped.\n\n---\n\n## DX Scorecard\n\n```\n+====================================================================+\n| DX PLAN REVIEW \u2014 SCORECARD |\n+====================================================================+\n| Dimension | Before | After | Trend |\n|----------------------|--------|--------|---------|\n| Getting Started | 2/10 | 8/10 | +6 \u2191 |\n| API/CLI/SDK | 5/10 | 9/10 | +4 \u2191 |\n| Error Messages | 5/10 | 9/10 | +4 \u2191 |\n| Documentation | 6/10 | 8/10 | +2 \u2191 |\n| Upgrade Path | 3/10 | 9/10 | +6 \u2191 |\n| Dev Environment | 8/10 | 8/10 | 0 \u2192 |\n| Community | 6/10 | 6/10 | 0 \u2192 |\n| DX Measurement | 8/10 | 8/10 | 0 \u2192 |\n+--------------------------------------------------------------------+\n| TTHW | 6 min | < 2 min| -4m \u2191 |\n| Competitive Rank | Needs Work \u2192 Champion (after T2 lands) |\n| Magical Moment | designed via terminal demo (python -m demo) |\n| Product Type | Library / SDK (Python) |\n| Mode | DX POLISH |\n| Overall DX | 5/10 | 8/10 | +3 \u2191 |\n+====================================================================+\n| DX PRINCIPLE COVERAGE |\n| Zero Friction | gap (FP1+FP2) \u2192 covered after T1+T2 |\n| Learn by Doing | covered (demo with real scores) |\n| Fight Uncertainty | gap (FP3) \u2192 covered after T3 |\n| Opinionated + Escape Hatches | gap (no CI skip) \u2192 covered (T2) |\n| Code in Context | covered (demo prints real output) |\n| Magical Moments | designed (terminal demo, real scores) |\n+====================================================================+\n```\n\n---\n\n## DX Implementation Checklist\n\n```\nDX IMPLEMENTATION CHECKLIST\n============================\n[x] Installation is one command \u2014 pip install evalkit==2.0.0b1\n[x] First run produces meaningful output \u2014 demo prints per-example + overall scores\n[x] Magical moment delivered via terminal demo\n[x] Every other error message has: problem + cause + fix \u2014 already done\n[x] Works in CI/CD without special configuration \u2014 noninteractive CI mode exists\n[x] Changelog exists and is maintained\n[x] Type annotations included\n\n[ ] T1: README \u2014 replace examples/first_eval.py reference with python -m evalkit.demo\n[ ] T2: Add --skip-ci-check / EVALKIT_SKIP_CI=1 flag \u2192 enables < 2 min TTHW\n[ ] T3: Update AuthError message: name cause + fix (not \"request failed\")\n[ ] T4: Add Client.evaluate() deprecation shim \u2192 Client.run() with DeprecationWarning\n[ ] T5: Standardize run_eval and run_batch to (dataset, evaluator) positional order\n[ ] T6: Add one-line CI check explanation at first progress message in demo\n[ ] T7: Write docs/migration-v1-v2.md \u2014 rename, grep command, shim removal timeline\n```\n\n---\n\n## Implementation Tasks\n\nSynthesized from this review's findings. P1 blocks ship; P2 should land same branch.\n\n- [ ] **T1 (P1, human: ~30min / CC: ~5min)** \u2014 README \u2014 Fix getting-started command\n - Surfaced by: Pass 1 / FP1 \u2014 `examples/first_eval.py` absent from published package\n - Files: `README.md`\n - Verify: `python -m evalkit.demo` is the canonical first command; no `examples/first_eval.py` reference remains\n\n- [ ] **T2 (P1, human: ~1 day / CC: ~30min)** \u2014 evalkit/demo.py + evalkit/client.py \u2014 Add `--skip-ci-check` flag\n - Surfaced by: Pass 1 / FP2 \u2014 mandatory 5-min CI wait is the entire TTHW gap vs. agreed target\n - Files: `evalkit/demo.py`, `evalkit/client.py`, `README.md`\n - Verify: `python -m evalkit.demo --skip-ci-check` completes in < 30s and prints scores; `EVALKIT_SKIP_CI=1` env var also works\n\n- [ ] **T3 (P1, human: ~1h / CC: ~10min)** \u2014 evalkit/client.py \u2014 Fix AuthError message\n - Surfaced by: Pass 3 / FP3 \u2014 only error in the SDK without cause or fix\n - Files: `evalkit/client.py`\n - Verify: With a bad API key, error message names the problem (invalid key), cause (key mismatch), and fix action\n\n- [ ] **T4 (P1, human: ~2h / CC: ~15min)** \u2014 evalkit/client.py \u2014 Add Client.evaluate() deprecation shim\n - Surfaced by: Pass 5 / FP4 \u2014 v1\u2192v2 silent breaking change, no warning\n - Files: `evalkit/client.py`\n - Verify: `client.evaluate(...)` raises `DeprecationWarning` naming `run()` as the replacement and still returns a valid result\n\n- [ ] **T5 (P1, human: ~2h / CC: ~15min)** \u2014 evalkit/client.py \u2014 Standardize run_eval/run_batch argument order\n - Surfaced by: Pass 2 / FP5 \u2014 reversed positional order causes silent wrong-result bugs\n - Files: `evalkit/client.py`, `evalkit/__init__.py`, API docs\n - Verify: Both `run_eval(dataset, evaluator)` and `run_batch(dataset, evaluator)` use identical positional order; all doc examples updated\n\n- [ ] **T6 (P2, human: ~30min / CC: ~5min)** \u2014 evalkit/demo.py \u2014 Add CI check explanation\n - Surfaced by: Pass 4 / TODO-1 \u2014 no context for 5-min wait\n - Files: `evalkit/demo.py`\n - Verify: First progress line reads something like: \"First-run quality gate: verifying sample integrity (~5 min). Skip with --skip-ci-check for local dev.\"\n\n- [ ] **T7 (P2, human: ~2h / CC: ~15min)** \u2014 docs/ \u2014 Write v1\u2192v2 migration guide\n - Surfaced by: Pass 5 / TODO-2 \u2014 no written migration guide for the planned breaking change\n - Files: `docs/migration-v1-v2.md`\n - Verify: Guide covers: what changed, what stayed the same, grep command for callsites, shim removal timeline (v2.1 target)\n\n---\n\n## GSTACK REVIEW REPORT\n\n| Review | Trigger | Why | Runs | Status | Findings |\n|--------|---------|-----|------|--------|----------|\n| CEO Review | `/plan-ceo-review` | Scope & strategy | 0 | \u2014 | \u2014 |\n| Outside Review | disabled | Independent 2nd opinion | 0 | disabled | codex_reviews=disabled |\n| Eng Review | `/plan-eng-review` | Architecture & tests (required) | 0 | \u2014 | \u2014 |\n| Design Review | `/plan-design-review` | UI/UX gaps | 0 | \u2014 | \u2014 |\n| DX Review | `/plan-devex-review` | Developer experience gaps | 1 | issues_open | score: 5/10 \u2192 8/10, TTHW: 6min \u2192 <2min |\n\n**OUTSIDE COVERAGE:** Codex disabled (codex_reviews=disabled). No outside voice this run.\n\n**VERDICT:** DX Review complete (issues_open \u2014 7 tasks specified) \u2014 eng review required\n\nNO UNRESOLVED DECISIONS\n", + "pendingExit": { + "sessionId": "c863dd11-6557-4988-b509-ac5229e24427", + "toolUseId": "toolu_019H5e2wALZGyJNX2Xf9SxCE", + "cwd": "/tmp/gstack-paid-shard-rVZvol/tmp/gstack-plan-count-u2oA0P", + "transcriptPath": "/tmp/gstack-paid-shard-rVZvol/tmp/gstack-hermetic-2786471-HlBxFP/with-skills/.claude/projects/-tmp-gstack-paid-shard-rVZvol-tmp-gstack-plan-count-u2oA0P/c863dd11-6557-4988-b509-ac5229e24427.jsonl", + "timestamp": "2026-09-09T12:14:38.022Z" + }, + "screen": "\n - Overall: 5/10 \u2192 8/10\n - TTHW: 6 min \u2192 < 2 min (Champion tier, after T2)\n - 5 P1 tasks, 2 P2 tasks\n\n GSTACK REVIEW REPORT\n\n \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n \u2502 Review \u2502 Trigger \u2502 Why \u2502 Runs \u2502 Status \u2502 Findings \u2502\n \u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n \u2502 CEO Review \u2502 /plan-ceo-review \u2502 Scope & strategy \u2502 0 \u2502 \u2014 \u2502 \u2014 \u2502\n \u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n \u2502 Outside \u2502 disabled \u2502 Independent 2nd \u2502 0 \u2502 disabled \u2502 codex_reviews=disabled \u2502\n \u2502 Review \u2502 \u2502 opinion \u2502 \u2502 \u2502 \u2502\n \u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n \u2502 Eng Review \u2502 /plan-eng-review \u2502 Architecture & tests \u2502 0 \u2502 \u2014 \u2502 \u2014 \u2502\n \u2502 \u2502 \u2502 (required) \u2502 \u2502 \u2502 \u2502\n \u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n \u2502 Design \u2502 /plan-design-review \u2502 UI/UX gaps \u2502 0 \u2502 \u2014 \u2502 \u2014 \u2502\n \u2502 Review \u2502 \u2502 \u2502 \u2502 \u2502 \u2502\n \u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n \u2502 DX Review \u2502 /plan-devex-review \u2502 Developer experience \u2502 1 \u2502 issues_open \u2502 score: 5/10 \u2192 8/10, TTHW: 6min \u2502\n \u2502 \u2502 \u2502 gaps \u2502 \u2502 \u2502 \u2192 <2min \u2502\n \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n\n OUTSIDE COVERAGE: Codex disabled (codex_reviews=disabled). No outside voice this run.\n\n VERDICT: DX Review complete (issues_open \u2014 7 tasks specified) \u2014 eng review required\n\n NO UNRESOLVED DECISIONS\n\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\n\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n Claude has written up a plan and is ready to execute. Would you like to proceed?\n\n \u276f 1. Yes, and use auto mode\n 2. Yes, manually approve edits\n 3. Tell Claude what to change\n shift+tab to approve with this feedback\n", + "provenance": { + "evidence": "/home/vercel-sandbox/gstack/.context/ship-source-z-full-paid-20260909-1158/devex-exit-audit-v1/proof.json", + "reportMtimeMs": 1788956000538.91, + "reportSha256": "86ba1fa4f10c4ce4bd2f6383d0dbfdbfa718b873532a963d8354454bbe477aca", + "capture": { + "skill": "plan-devex-review", + "runId": "ship-source-z-full-paid-20260909-1158-5", + "cwd": "/tmp/gstack-paid-shard-rVZvol/tmp/gstack-plan-count-u2oA0P", + "claudeConfigDir": "/tmp/gstack-paid-shard-rVZvol/tmp/gstack-hermetic-2786471-HlBxFP/with-skills/.claude", + "at": "2026-09-09T12:23:10.424Z" + }, + "startedAt": "2026-09-09T12:00:28Z", + "rawCounts": { + "step0Count": 3, + "reviewCount": 6, + "administrativeCount": 0 + } + } +} diff --git a/test/fixtures/eng-binding-retry-z-calls.json b/test/fixtures/eng-binding-retry-z-calls.json new file mode 100644 index 000000000..da3762b0e --- /dev/null +++ b/test/fixtures/eng-binding-retry-z-calls.json @@ -0,0 +1,182 @@ +[ + { + "sessionId": "db9c3882-1a7f-47c7-827e-16fc3fdf4031", + "toolUseId": "toolu_012Ro9wjqWQHApDSVPwATL21", + "questions": [ + { + "question": "D1 — Scope check: 12 files + 4 new classes exceeds the complexity threshold. Should we recommend reducing scope, or accept the plan as-is and review it at full size? ", + "header": "Scope check", + "multiSelect": false, + "options": [ + { + "label": "Recommend scope reduction (Recommended)", + "description": "Flag the complexity in the review and propose a minimal design: 2 new classes (AuthBroker + RequestPolicy), reuse the existing adapter as TokenStore, inline session minting into AuthBroker. Estimated ~6 files instead of 12. The review report will include a concrete scope-reduction recommendation the team can choose to adopt." + }, + { + "label": "Accept full scope, review as-is", + "description": "The 12-file, 4-class plan is accepted as the target. The review proceeds across Architecture, Code Quality, Tests, and Performance at full scope. Scope complexity is noted but not challenged." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D1 — Scope check: 12 files + 4 new classes exceeds the complexity threshold. Should we recommend reducing scope, or accept the plan as-is and review it at full size? ": "Recommend scope reduction (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:19:21.399Z" + }, + { + "sessionId": "db9c3882-1a7f-47c7-827e-16fc3fdf4031", + "toolUseId": "toolu_014Ty9SbhubQaW9QaD8UPL5d", + "questions": [ + { + "question": "D2 — Architecture: Two services share a global mutable AuthCache via module-level export. This is the #1 reliability risk in multi-tenant auth — concurrent mutations can corrupt tenant isolation. How should the plan address this? ", + "header": "Shared cache", + "multiSelect": false, + "options": [ + { + "label": "Dependency injection (Recommended)", + "description": "The plan is updated to pass AuthCache as a constructor argument to both AuthBroker and SessionMint. No module-level mutable export. Tests can inject a mock/stub. Single shared instance still possible at the app root. Completeness: 9/10." + }, + { + "label": "Mutation guards on the global", + "description": "Keep the global export but wrap every mutation site in explicit locking or compare-and-swap. Safer than bare shared state but still couples both services to the global. Adds concurrency primitives that need their own tests. Completeness: 7/10." + }, + { + "label": "Accept as-is, flag as known risk", + "description": "Note the shared-global pattern in the review report as a known risk. Leave the plan unchanged. Suitable only if the runtime is single-threaded and concurrent mutation is architecturally impossible. Completeness: 4/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D2 — Architecture: Two services share a global mutable AuthCache via module-level export. This is the #1 reliability risk in multi-tenant auth — concurrent mutations can corrupt tenant isolation. How should the plan address this? ": "Dependency injection (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:19:49.487Z" + }, + { + "sessionId": "db9c3882-1a7f-47c7-827e-16fc3fdf4031", + "toolUseId": "toolu_01C6JJGSvUTP8Gi9tcj8N4C9", + "questions": [ + { + "question": "D3 — Code quality: validateAndDispatch() has 3 nested try/catch blocks, each swallowing a different error class. Silent auth failures are a serious observability and reliability gap. What should the plan prescribe? ", + "header": "Error handling", + "multiSelect": false, + "options": [ + { + "label": "Re-throw as typed domain errors (Recommended)", + "description": "Each catch block catches its error class, logs it, then re-throws a typed domain error (e.g. TokenValidationError, PolicyError). Callers get structured errors. The function shrinks to ~30 lines. Full observability + type safety. Completeness: 9/10." + }, + { + "label": "Single top-level handler + flatten", + "description": "Extract the three try/catch bodies into separate private methods, wrap the whole function in one top-level try/catch that logs and re-throws. Simpler than per-error typing but loses the error-class distinction callers may need. Completeness: 7/10." + }, + { + "label": "Accept current structure", + "description": "Note the error-swallowing as a code smell in the report, but don't prescribe a change in the plan. Suitable if error types are genuinely irrelevant to callers and observability is handled elsewhere. Completeness: 4/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D3 — Code quality: validateAndDispatch() has 3 nested try/catch blocks, each swallowing a different error class. Silent auth failures are a serious observability and reliability gap. What should the plan prescribe? ": "Re-throw as typed domain errors (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:20:01.524Z" + }, + { + "sessionId": "db9c3882-1a7f-47c7-827e-16fc3fdf4031", + "toolUseId": "toolu_01HLXRnjNyqRXBUUWy9NQYq4", + "questions": [ + { + "question": "D4 — Tests: The D2 decision introduced DI for AuthCache. Should the test plan explicitly require a concurrent-mutation test — two requests from different tenants hitting AuthBroker and SessionMint simultaneously — to verify tenant isolation holds under load? ", + "header": "Concurrency test", + "multiSelect": false, + "options": [ + { + "label": "Add concurrent-mutation test (Recommended)", + "description": "Add a test that fires AuthBroker and SessionMint simultaneously with different tenant IDs and asserts no cross-tenant cache bleed. This is the failure mode DI was chosen to prevent; testing it closes the loop. Completeness: 9/10." + }, + { + "label": "Defer — unit tests cover the contract", + "description": "Unit tests for the DI'd AuthCache interface plus the existing adapter tests are sufficient. Concurrent integration testing can be added post-ship if production issues surface. Completeness: 6/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D4 — Tests: The D2 decision introduced DI for AuthCache. Should the test plan explicitly require a concurrent-mutation test — two requests from different tenants hitting AuthBroker and SessionMint simultaneously — to verify tenant isolation holds under load? ": "Add concurrent-mutation test (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:20:43.682Z" + }, + { + "sessionId": "db9c3882-1a7f-47c7-827e-16fc3fdf4031", + "toolUseId": "toolu_01LHzWK7Mzr4tT2iFN3Q3tzY", + "questions": [ + { + "question": "D5 — Performance: Token validation makes 5 sequential IDP calls that are independent. Should the plan prescribe parallelizing them via Promise.all, or defer? ", + "header": "IDP parallelism", + "multiSelect": false, + "options": [ + { + "label": "Parallelize in this PR (Recommended)", + "description": "Replace sequential IDP calls with Promise.all. The plan itself calls this trivial and the calls are confirmed independent. ~5x auth latency improvement. Error handling: Promise.allSettled if partial success is acceptable, Promise.all if all-or-nothing. Add a test for the parallel execution path. Completeness: 9/10." + }, + { + "label": "Defer to a follow-up PR", + "description": "Merge the refactor first, parallelize in a separate PR. Lower risk per-PR, but ships known 5x latency regression compared to what is achievable. Completeness: 6/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D5 — Performance: Token validation makes 5 sequential IDP calls that are independent. Should the plan prescribe parallelizing them via Promise.all, or defer? ": "Parallelize in this PR (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:20:53.719Z" + }, + { + "sessionId": "db9c3882-1a7f-47c7-827e-16fc3fdf4031", + "toolUseId": "toolu_01JywMpehKe8eWcfqbYKDyfb", + "questions": [ + { + "question": "D6 — TODO: Should we capture a cross-tenant E2E integration test (real IDP, two tenants, verify no cache bleed) as a TODO for post-ship? The D4 concurrent-mutation test covers the unit level; this covers the full integration boundary. ", + "header": "TODO: E2E test", + "multiSelect": false, + "options": [ + { + "label": "Add to TODOS.md (Recommended)", + "description": "Capture the cross-tenant E2E test as a tracked TODO. Why: the unit-level concurrent test covers cache isolation; the E2E test covers the full flow including IDP, policy enforcement, and session creation across tenants. Prerequisites: test IDP infrastructure." + }, + { + "label": "Skip — unit tests are sufficient", + "description": "The concurrent-mutation test plus unit/integration coverage the plan already describes is enough. Don't add an E2E TODO." + }, + { + "label": "Build it now in this PR", + "description": "Include the cross-tenant E2E test in the current PR scope. Expands scope but closes the full integration gap now." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D6 — TODO: Should we capture a cross-tenant E2E integration test (real IDP, two tenants, verify no cache bleed) as a TODO for post-ship? The D4 concurrent-mutation test covers the unit level; this covers the full integration boundary. ": "Add to TODOS.md (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:21:45.933Z" + } +] diff --git a/test/fixtures/eng-binding-z-calls.json b/test/fixtures/eng-binding-z-calls.json new file mode 100644 index 000000000..0d4ab67f7 --- /dev/null +++ b/test/fixtures/eng-binding-z-calls.json @@ -0,0 +1,182 @@ +[ + { + "sessionId": "9b477fa1-a8ff-463d-9863-0c63a3a79b48", + "toolUseId": "toolu_01NXbqfKxF1qxpbYAThLzzJY", + "questions": [ + { + "question": "D1 — Scope challenge: 12 files + 4 new classes exceeds the complexity threshold. Should we reduce scope or proceed as-is? ", + "header": "Scope", + "multiSelect": false, + "options": [ + { + "label": "Reduce scope (Recommended)", + "description": "Cut to 1-2 new classes: merge TokenStore + SessionMint into AuthBroker; use the existing cache adapter directly instead of a new AuthCache facade; fold RequestPolicy into AuthBroker as a private method. Target: ~6-8 files, 1-2 classes. Same outcome, less surface area to test and maintain. (human: ~1 day saved / CC: ~10min to restructure)" + }, + { + "label": "Proceed as-is", + "description": "Keep all 4 new classes across 12 files. The plan argues each class has a distinct responsibility. Proceed to full review of the existing design without scope reduction. (human: no change / CC: no change)" + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D1 — Scope challenge: 12 files + 4 new classes exceeds the complexity threshold. Should we reduce scope or proceed as-is? ": "Reduce scope (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:10:13.579Z" + }, + { + "sessionId": "9b477fa1-a8ff-463d-9863-0c63a3a79b48", + "toolUseId": "toolu_01Hg4GPH9pzVWAV51s6Rgx4q", + "questions": [ + { + "question": "D2 — Architecture: How should AuthBroker access the cache adapter after scope reduction? ", + "header": "Cache binding", + "multiSelect": false, + "options": [ + { + "label": "Constructor injection (Recommended)", + "description": "AuthBroker receives the cache adapter as a constructor argument (or factory function parameter). Tests pass a stub; production passes the real adapter. Eliminates module-level mutable state entirely. Requires wiring at the call site. Completeness: 9/10. (human: ~2h / CC: ~5min)" + }, + { + "label": "Module-level import", + "description": "AuthBroker imports the adapter directly at module scope, same pattern as the current plan. Works fine in production; makes tests require module-level mocking (jest.mock, proxyquire). Matches the existing codebase pattern if that's what's already used. Completeness: 6/10. (human: no change / CC: no change)" + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D2 — Architecture: How should AuthBroker access the cache adapter after scope reduction? ": "Constructor injection (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:11:05.687Z" + }, + { + "sessionId": "9b477fa1-a8ff-463d-9863-0c63a3a79b48", + "toolUseId": "toolu_01FsTi5TZqg3CxiwxeyeAkvt", + "questions": [ + { + "question": "D3 — Code quality: validateAndDispatch() swallows errors. How should we handle error propagation? ", + "header": "Error handling", + "multiSelect": false, + "options": [ + { + "label": "Refactor + typed errors (Recommended)", + "description": "Extract the three catch blocks into named helper functions. Replace swallow behavior with explicit typed error returns (discriminated union) or re-throw with context. validateAndDispatch() becomes a 15-line orchestrator. Each error type is explicitly handled, observable, and testable. Completeness: 9/10. (human: ~3h / CC: ~10min)" + }, + { + "label": "Refactor structure only", + "description": "Extract into helpers to reduce nesting, but keep the swallow behavior for now (existing contract). Reduces cognitive load without changing observable behavior. Can add typed errors in a follow-up. Completeness: 7/10. (human: ~1h / CC: ~5min)" + }, + { + "label": "Leave as-is", + "description": "Document the swallow behavior with a comment. Tech debt acknowledged; fix in follow-up. Low effort but the silent failures remain. Completeness: 3/10. (human: 15min / CC: 1min)" + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D3 — Code quality: validateAndDispatch() swallows errors. How should we handle error propagation? ": "Refactor + typed errors (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:11:19.707Z" + }, + { + "sessionId": "9b477fa1-a8ff-463d-9863-0c63a3a79b48", + "toolUseId": "toolu_01D1bqdZYF7NhzYAT8mtqWqG", + "questions": [ + { + "question": "D4 — Tests: What is the desired behavior when k of 5 parallel IDP calls fail? This decides the error path test case. ", + "header": "IDP partial fail", + "multiSelect": false, + "options": [ + { + "label": "Fail fast — any failure fails validation (Recommended)", + "description": "If any of the 5 IDP calls fails, the whole token validation fails with a typed TokenValidationError. Safest security posture: one failed check = reject. Tests cover: 1-fail, all-fail, all-succeed. Completeness: 9/10. (human: ~30min to specify / CC: ~5min)" + }, + { + "label": "Partial success — all 5 must succeed", + "description": "Same outcome as fail-fast semantically (all must succeed), but documents the all-must-succeed requirement explicitly in the plan so test cases are unambiguous. Clarifies intent for future readers without changing behavior. Completeness: 9/10. (human: ~15min / CC: ~2min)" + }, + { + "label": "Best effort — majority must succeed", + "description": "If N-of-5 succeed, token passes. Threshold configurable. More complex logic; rare in auth systems because partial validity is a security gap. Completeness: 7/10. (human: ~1 day / CC: ~30min)" + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D4 — Tests: What is the desired behavior when k of 5 parallel IDP calls fail? This decides the error path test case. ": "Fail fast — any failure fails validation (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:12:03.814Z" + }, + { + "sessionId": "9b477fa1-a8ff-463d-9863-0c63a3a79b48", + "toolUseId": "toolu_01QmbWT9eYiGT7vUJgeeXett", + "questions": [ + { + "question": "D5 — Performance: Parallelize the 5 sequential IDP calls via Promise.all in this PR or defer? ", + "header": "IDP parallel", + "multiSelect": false, + "options": [ + { + "label": "Parallelize now (Recommended)", + "description": "Replace sequential await chain with Promise.all([call1, call2, call3, call4, call5]). The plan already notes this is trivial and calls are independent. Reduces auth latency by ~4x in practice. Fail-fast semantics from D4 work cleanly with Promise.all rejection. Add one test: all-fail triggers rejection. (human: ~30min / CC: ~3min)" + }, + { + "label": "Defer to follow-up", + "description": "Leave sequential for now. Document as a known perf gap in TODOS.md. Low risk of blocking ship. Revisit when auth latency becomes a measured issue. (human: ~5min to document / CC: ~1min)" + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D5 — Performance: Parallelize the 5 sequential IDP calls via Promise.all in this PR or defer? ": "Parallelize now (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:12:19.838Z" + }, + { + "sessionId": "9b477fa1-a8ff-463d-9863-0c63a3a79b48", + "toolUseId": "toolu_014kJdUP73WGRwuSvtstZapy", + "questions": [ + { + "question": "D6 — TODOS: Add IDP call timeout protection to TODOS.md? ", + "header": "TODO: Timeout", + "multiSelect": false, + "options": [ + { + "label": "Add to TODOS.md (Recommended)", + "description": "Log a TODO: wrap each parallel IDP call with an explicit timeout (AbortSignal or Promise.race). Without it, a slow IDP silently blocks auth for all users behind that tenant. The plan has no timeout handling and no test for it — this is the one remaining critical gap. Capture context now while it's fresh. (human: ~15min / CC: ~2min to add TODO)" + }, + { + "label": "Build it now in this PR", + "description": "Add timeout wrapping to AuthBroker's IDP call logic in this PR. Adds ~20-30 lines; needs a configurable threshold and a test for the timeout path. This is the complete version. (human: ~1h / CC: ~10min)" + }, + { + "label": "Skip", + "description": "Don't capture this. Accept the risk of silent auth hangs on slow IDP endpoints. Only reasonable if the IDP is always fast by contract (e.g., same-datacenter, sub-10ms). (human: ~0 / CC: ~0)" + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D6 — TODOS: Add IDP call timeout protection to TODOS.md? ": "Add to TODOS.md (Recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-09T12:13:17.972Z" + } +] diff --git a/test/helpers/auq-sdk-capture.ts b/test/helpers/auq-sdk-capture.ts index 4b3f3e626..58a7dfdcd 100644 --- a/test/helpers/auq-sdk-capture.ts +++ b/test/helpers/auq-sdk-capture.ts @@ -271,7 +271,7 @@ export async function captureSectionReads(opts: { // Preserve full method execution while avoiding a second written walkthrough // of decisions already represented in the amended plan and required outputs. const planReviewWritingRule = opts.skillName === 'plan-ceo-review' - ? `\n- Write a concise, complete decision record: preserve original requirements and accepted plan amendments. Record each finding once with concrete evidence, the selected remedy, residual risks, and verification. Give all 11 sections an explicit outcome (including no issues or justified skips); retain the complete required registries, applicable diagrams, tasks, completion summary, and exact GSTACK REVIEW REPORT table. Cross-reference those records instead of repeating findings, option deliberations, diagrams, or registries in each section. Do not expand the artifact into full implementation or test code unless that code is needed to specify an accepted plan change. This is a writing rule only: execute the full review, perform every required lazy-file Read, and complete all required artifacts before returning.` + ? `\n- Write a concise, complete decision record: preserve original requirements and accepted plan amendments. Record each finding once with concrete evidence, the selected remedy, residual risks, and verification. Give all 11 sections an explicit outcome (including no issues or justified skips); retain the complete required registries, applicable diagrams, tasks, completion summary, and exact GSTACK REVIEW REPORT table. Cross-reference those records instead of repeating findings, option deliberations, diagrams, or registries in each section. Use compact outcome entries and short table cells; execute the review checklists without copying their questions or narrating every check into the artifact. Brevity must preserve every finding, accepted requirement, required field, and required diagram in its specified format. Do not expand the artifact into full implementation or test code unless that code is needed to specify an accepted plan change. This is a writing rule only: execute the full review, perform every required lazy-file Read, and complete all required artifacts before returning.` : ''; const prompt = `You are running an automated skill-execution test. No human is present, so AskUserQuestion is unavailable. The ONLY skill file you may read is this absolute path: ${skillPath}. Do NOT Glob/find/search for any other SKILL.md anywhere — especially nothing under ~/.claude or /Users. diff --git a/test/helpers/autoplan-setup-question.ts b/test/helpers/autoplan-setup-question.ts index cb899368c..cf0ad197f 100644 --- a/test/helpers/autoplan-setup-question.ts +++ b/test/helpers/autoplan-setup-question.ts @@ -1,6 +1,6 @@ import { capturePlanCountQuestion, parseNumberedOptions, planCountPrerequisitePick, planCountQuestionInput, planCountSubmissionInput, type AskUserQuestionFingerprint } from './claude-pty-runner'; -import type { NativePlanQuestionCall } from './plan-count-transcript'; +import type { NativePlanQuestion, NativePlanQuestionCall } from './plan-count-transcript'; /** A copied native panel is not actionable after prose or inside a code example. */ function activeSetupPanel(visible: string): boolean { @@ -132,6 +132,36 @@ function routingSetupActions(question: AskUserQuestionFingerprint, allowTemporar return { add, decline }; } +/** Numbered setup wording may vary; newly admitted forms still consume every description. */ +function numberedPacketSetup(question: NativePlanQuestion, pending: NativePlanQuestionCall) { + if (pending.answered !== false || pending.failed !== false || !pending.sessionId || !pending.toolUseId || question.options.length !== 2) return null; + const compact = (value: string | undefined) => (value ?? '').trim().replace(/\s+/g, ' '); + const ids = [...question.question.matchAll(//gi)]; + const text = compact(question.question.replace(//gi, '')) + .replace(/^D[1-9]\d*\s*[—–:-]\s*/, ''); + const fp: AskUserQuestionFingerprint = { signature: '', observedAtMs: 0, preReview: true, + promptSnippet: `${question.header} ${question.question}`, + options: question.options.map((option, index) => ({ index: index + 1, label: option.label })) }; + const routing = routingSetupActions(fp, true); + if (ids.length === 1 && ids[0]![0].toLowerCase() === '' && + /^Add\s+(?:gstack\s+)?skill\s+routing\s+rules\s+to\s+CLAUDE\.md\?$/i.test(text) && + routing?.add.length === 1 && routing.decline.length === 1 && routing.add[0]!.index !== routing.decline[0]!.index) { + const add = question.options[routing.add[0]!.index - 1]!; + const decline = question.options[routing.decline[0]!.index - 1]!; + if (/^Appends a skill routing section to CLAUDE\.md so future sessions automatically invoke the right skill(?: \(e\.g\. \/autoplan for reviews, \/ship for deploys\))? without you needing to type the command each time\. One-time setup per project\.$/i.test(compact(add.description)) && + /^No change to CLAUDE\.md\. You['’]ll continue calling skills yourself with \/skill-name as you do now\.$/i.test(compact(decline.description))) { + return { kind: 'routing', pick: routing.add[0]!.index }; + } + } + if (ids.length || !/^No design doc (?:found|exists) for (?:this|the) (?:branch|project)\. Run \/office-hours (?:first|now)(?: to sharpen (?:the|this) review input)?\?$/i.test(text)) return null; + const run = question.options.findIndex(option => /^Run\s*\/office-hours\s*(?:now|first)(?:\s*\(recommended\))?$/i.test(option.label)); + const skip = question.options.findIndex(option => /^Skip\s*[,—–-]\s*proceed\s+with\s+(?:standard\s+)?review(?:\s*\(recommended\))?$/i.test(option.label)); + if (run < 0 || skip < 0 || run === skip || + !/^Start the full CEO (?:→|->) Design (?:→|->) DX (?:→|->) Eng review pipeline now using the plan as-is\. (?:Recommended when the plan context is already rich enough\. ?)?(?:\(Recommended\))?$/i.test(compact(question.options[skip]!.description)) || + !/^Produces a structured problem statement, premise challenge, and explored alternatives before the review\. (?:Takes ~?\d+(?:[–-]\d+)? min\. )?Gives the review sharper, better-grounded input\.$/i.test(compact(question.options[run]!.description))) return null; + return { kind: 'prerequisite', pick: skip + 1 }; +} + /** Answer only the known pair of setup offers, using the actual native active tab. */ function setupPacketDecision(visible: string, seen: ReadonlySet, pending: NativePlanQuestionCall): AutoplanSetupDecision { const waiting: AutoplanSetupDecision = { kind: 'waiting' }; @@ -159,7 +189,7 @@ function setupPacketDecision(visible: string, seen: ReadonlySet, pending // Require an actual prerequisite offer, not a product question that // happens to mention the absence of an office-hours design document. const offer = /^No\s+design\s+doc\s+(?:found|exists)(?:\s+for\s+(?:this|the)\s+(?:branch|project))?\.\s*(?:\/office-hours\s+(?:produces|creates|provides)\s+(?:a\s+)?(?:structured\s+)?(?:design\s+doc(?:ument)?|problem\s+statement)(?:,?\s+(?:and\s+)?(?:premise\s+challenge|(?:explored\s+)?alternatives))*(?:\s*[—–-]\s*(?:sharper|better)\s+input\s+for\s+(?:the|this)\s+review)?\.\s*)?(?:Want\s+to\s+|Would\s+you\s+like\s+to\s+)?Run\s+(?:it|\/office-hours)\s+(?:now|first)(?:\s+or\s+proceed\s+with\s+standard\s+review)?\s*\?$/i.test(offerText); - return prerequisite !== null && run.length === 1 && offer ? { kind: 'prerequisite', pick: prerequisite } : null; + return prerequisite !== null && run.length === 1 && offer ? { kind: 'prerequisite', pick: prerequisite } : numberedPacketSetup(question, pending); }); if (policies.some(policy => !policy) || new Set(policies.map(policy => policy!.kind)).size !== 2) return waiting; diff --git a/test/helpers/ceo-completion-handoff.ts b/test/helpers/ceo-completion-handoff.ts index d8b7b57cd..fe2f92000 100644 --- a/test/helpers/ceo-completion-handoff.ts +++ b/test/helpers/ceo-completion-handoff.ts @@ -175,6 +175,34 @@ function bareEngNavigation(question: string, descriptions: string[]): boolean { /^Eng review is the required shipping gate [—–-] skipping it means less confidence before enabling the feature flag$/i.test(s)); } +/** A completed CEO review may distinguish the still-unrun Eng shipping gate. */ +function unrunEngNavigation(fp: AskUserQuestionFingerprint, question: string): number | null { + const call = fp.nativeCall!; + const q = call.questions[0]!; + if (!call.sessionId || !call.toolUseId || call.failed !== false || + (fp.nativeQuestionIndex !== undefined && fp.nativeQuestionIndex !== 0) || + q.options.length !== 2 || fp.options.length !== 2 || + !fp.options.every((o, i) => o.index === i + 1 && o.label === q.options[i]!.label) || + !/^Next review$/i.test(q.header.trim()) || + !/^CEO Review is CLEAR\. Eng Review is the required shipping gate and (?:hasn['’]t|has not) run yet\. What(?:['’]s| is) next\?$/i.test(question)) return null; + if (call.answered === false) { + if (call.answers !== undefined || call.answeredAt !== undefined || + (call.unansweredQuestionIndices !== undefined && + (call.unansweredQuestionIndices.length !== 1 || call.unansweredQuestionIndices[0] !== 0))) return null; + } else if (call.answered !== true || !Array.isArray(call.unansweredQuestionIndices) || call.unansweredQuestionIndices.length) return null; + const labels = q.options.map(o => o.label.trim().replace(/^[A-Z][).]\s*/i, '').replace(/\s*\(recommended\)\s*$/i, '').trim()); + const run = labels.findIndex(s => /^Run \/plan-eng-review(?: next| now)?$/i.test(s)); + const manual = labels.findIndex(s => /^Skip\s*[—–-]\s*I['’]ll handle reviews manually$/i.test(s)); + if (run < 0 || manual < 0 || run === manual) return null; + const topics = String.raw`(?:architecture|code quality|test design|performance|deployment)`; + const runDescription = new RegExp(String.raw`^${topics}(?:,\s+${topics})*(?:,?\s+and\s+${topics})?\s+review\.\s+The required gate before shipping\.\s+Run this before implementation begins to catch any structural issues in how the tests are wired up\.$`, 'i'); + // Consume each complete description in its own offered role. The temporal + // qualification is about the next review, not an unfinished CEO decision. + if (!runDescription.test(q.options[run]!.description?.trim() ?? '') || + !/^Proceed to implementation directly\.\s+You can run \/plan-eng-review later if needed\.\s+Eng Review is required before shipping but not before starting implementation\.$/i.test(q.options[manual]!.description?.trim() ?? '')) return null; + return manual + 1; +} + /** Shared closed-review guards; next-review sequencing is still navigation. */ function closedNavigationContext(context: string): boolean { const unfinished = context.replace(/\b(?:no|0)\s+unresolved\s+(?:decisions|gaps|issues|findings)\b/gi, ''); @@ -208,6 +236,8 @@ function manualHandoffIndex(fp: AskUserQuestionFingerprint): number | null { const explicitCompletion = /(?:^|[.!?]\s+)(?:ELI10:\s*)?(?:The\s+)?CEO\s+review\s+(?:is\s+)?(?:complete|cleared|clean|done(?:\s+and\s+the\s+plan\s+is\s+cleared)?)(?:\s+with\s+0\s+unresolved\s+decisions)?(?=\s*(?:[.!?—–]|$))/i.test(declaration); const genericCompletion = /(?:^|[.!?]\s+)(?:The\s+)?review\s+(?:is\s+)?(?:complete|cleared|clean|done)(?=\s*(?:[.!?—–]|$))/i.test(declaration); const questionText = declaration.replace(/]+>/gi, '').trim(); + const unrunNavigation = id ? unrunEngNavigation(fp, questionText) : null; + if (unrunNavigation !== null) return unrunNavigation; const recappedNavigation = Boolean(id) && /^What(?:['’]s|\s+is)\s+the\s+next\s+(?:steps?|review)\s+after\s+(?:this|the)\s+CEO\s+review\?$/i.test(questionText) && q.options.some(option => resolvedCeoRecap(option.description ?? '')); diff --git a/test/helpers/claude-pty-runner.ts b/test/helpers/claude-pty-runner.ts index 5b4b5fbbc..7634e4a14 100644 --- a/test/helpers/claude-pty-runner.ts +++ b/test/helpers/claude-pty-runner.ts @@ -1370,6 +1370,38 @@ export function planCountQuestionInput(visible: string, fp: AskUserQuestionFinge return panel ? String(index) : `${index}\r`; } +/** A native single-question pane can elide its tail to leave room for choices. */ +function matchesTruncatedNativeQuestion(visible: string, call: NativePlanQuestionCall): boolean { + if (call.questions.length !== 1) return false; + const cursor = [...visible.matchAll(/❯\s*1\./g)].at(-1); + if (!cursor) return false; + const before = visible.slice(0, cursor.index); + const header = /^(?:[\t │┃]*\n)*[\t │┃]*[☐□]([^\n│]*)\n/.exec(before); + if (!header || /[☐□❯]/.test(before.slice(header[0].length))) return false; + const exact = (value: string) => value.replace(/\s+/g, ''); + const question = call.questions[0]!; + if (exact(header[1]!) !== exact(question.header)) return false; + const body = before.slice(header[0].length).replace(/^[ \t]*[│┃] ?|[│┃][ \t]*$/gm, '').trim(); + // Require the complete displayed prefix, not a fragment or common heading. + // The final ellipsis is the native UI's elision indicator; missing or changed + // text anywhere before it cannot borrow an unrelated call's answer policy. + if (!body.endsWith('…')) return false; + const prefix = exact(body.slice(0, -1)); + const native = exact(question.question); + if (prefix.length < 160 || body.split('\n').filter(line => line.trim()).length < 2 || + prefix.length >= native.length || !native.startsWith(prefix)) return false; + const menu = visible.slice(cursor.index); + const footer = /Enter\s*to\s*select\s*·\s*↑\/↓\s*to\s*navigate\s*·\s*(?:n\s*to\s*add\s*notes\s*·\s*)?Esc\s*to\s*cancel/i.exec(menu); + if (!footer || !/^[\s│┃─━└┘]*$/.test(menu.slice(footer.index + footer[0].length))) return false; + const options = parseNumberedOptions(visible); + const offered = options.slice(0, question.options.length); + const controls = options.slice(question.options.length); + return offered.length === question.options.length && offered.every((option, index) => + option.index === index + 1 && exact(option.label) === exact(question.options[index]!.label)) && + controls.length <= 2 && controls.every((option, index) => option.index === question.options.length + index + 1 && + (index === 0 ? /^Typesomething\.?$/ : /^Chataboutthis$/).test(exact(option.label))); +} + /** Match native question identity before permission text can choose an answer. */ export function matchesNativePlanQuestion(visible: string, call: NativePlanQuestionCall): boolean { if (call.failed) return false; @@ -1385,7 +1417,7 @@ export function matchesNativePlanQuestion(visible: string, call: NativePlanQuest if (!header) return matchesClippedNativeQuestion(normalized, call); if (compact(header[1]!) !== compact(question.header) || /❯\s*[1-9]\./.test(before.slice(header.index))) return false; const identity = question.question.match(/]+>/i)?.[0] ?? question.question; - if (!compact(before.slice(header.index)).includes(compact(identity))) return false; + if (!compact(before.slice(header.index)).includes(compact(identity))) return matchesTruncatedNativeQuestion(normalized, call); // Preserve the captured damaged-option path when the native panel's // complete footer is intact, including the optional native preview notes key. // With a damaged footer, require the full @@ -1851,7 +1883,7 @@ function isCompletedDxHandoff(call: NativePlanQuestionCall): boolean { if (!call.answered || call.failed || call.questions.length !== 1 || !Array.isArray(call.unansweredQuestionIndices) || call.unansweredQuestionIndices.length) return false; const q = call.questions[0]!; - if (resolvedDxTaskNavigation(call)) return true; + if (resolvedDxTaskNavigation(call) || specifiedDxTaskNavigation(call)) return true; const header = q.header.trim().replace(/^D\s*\d+\s*(?:[—–:-]\s*)?/i, ''); const question = q.question.replace(/^D\s*\d+\s*[—–:-]\s*/i, ''); const completionLines: string[] = []; @@ -1920,6 +1952,30 @@ function isCompletedDxHandoff(call: NativePlanQuestionCall): boolean { q.options.some(o => o.label === call.answers?.[q.question]); } +/** A completed DX recap can offer navigation over already specified tasks. */ +function specifiedDxTaskNavigation(call: NativePlanQuestionCall): boolean { + const q = call.questions[0]!; + if (call.answered !== true || call.failed !== false || !call.sessionId || !call.toolUseId || + q.multiSelect || q.header.trim() !== 'Next steps' || q.options.length !== 3 || + new Set(q.options.map(o => o.label)).size !== 3 || + Object.keys(call.answers ?? {}).length !== 1) return false; + const compact = (value: string | undefined) => (value ?? '').replace(/\s+/g, ' ').trim(); + const question = compact(q.question); + const match = /^D\d+ [—–-] What's next\? DX review complete\. ([1-9]\d*) tasks specified \(([1-9]\d*) P1 block ship, ([1-9]\d*) P2 same branch\)\. DX score: (?:10|[0-9])\/10 → (?:10|[0-9])\/10\. TTHW: \d+(?:\.\d+)? min → < (\d+(?:\.\d+)?) min \(Champion tier\) after T\d+ lands\. The API changes \((T\d+, T\d+, T\d+)\) have architectural implications that warrant an eng review\. $/.exec(question); + if (!match || Number(match[1]) !== Number(match[2]) + Number(match[3]) || Number(match[4]) <= 0) return false; + const labels = q.options.map(o => o.label.trim().replace(/\s*\(Recommended\)$/, '')); + const expected = ['Run /plan-eng-review next', 'Ready to implement', 'Skip — handle next steps manually']; + const descriptions = [ + `API changes (${match[5]}) have architectural implications. Eng review validates the --skip-ci-check design, deprecation shim contract, and argument-order change scope before implementation.`, + `Skip eng review. Start implementing T1–T${match[1]} directly. Run /devex-review after shipping to measure the real TTHW against the < ${match[4]} min target.`, + 'No follow-up review needed right now.', + ]; + return labels.every((label, index) => { + const role = expected.indexOf(label); + return role >= 0 && compact(q.options[index]!.description) === descriptions[role]; + }) && new Set(labels).size === 3 && q.options.some(o => o.label === call.answers?.[q.question]); +} + /** Completed issue decisions can hand off their existing numbered tasks. */ function resolvedDxTaskNavigation(call: NativePlanQuestionCall): boolean { const q = call.questions[0]!; @@ -2168,9 +2224,68 @@ function engArchitectureChoiceAUQ(fp: AskUserQuestionFingerprint): boolean { return Boolean(unchanged && /^Each worker gets its own copy of the retry logic\.\s+Completeness:\s*\d+\/10\.\s+Creates \d+ divergence points; acknowledged DRY violation from the start\.$/i.test(unchanged.description ?? '')); } +/** A closed dependency choice can expose the current plan's coupling in its options. */ +function engDependencyBindingAUQ(fp: AskUserQuestionFingerprint): boolean { + const call = fp.nativeCall; + if (call?.answered !== true || call.failed !== false || call.questions.length !== 1 || + !Array.isArray(call.unansweredQuestionIndices) || call.unansweredQuestionIndices.length || + fp.signature !== `${call.sessionId}:${call.toolUseId}`) return false; + const q = call.questions[0]!; + if (q.multiSelect || !/^Cache binding$/i.test(q.header.trim()) || q.options.length !== 2 || + new Set(q.options.map(o => o.label)).size !== 2 || + q.options.filter(o => o.label === call.answers?.[q.question]).length !== 1 || + fp.options.length !== 2 || !fp.options.every((o, i) => o.index === i + 1 && o.label === q.options[i]!.label)) return false; + const ids = [...q.question.matchAll(/]+)>/gi)]; + if (ids.length !== 1 || (q.question.match(/ s.trim().replace(/\s+/g, ' '); + const body = normalize(q.question.replace(/\s*]+>\s*$/i, '')); + const choice = /^D\s*\d+\s*[—–:-]\s*Architecture: How should ([A-Za-z_$][\w$]*) access the cache adapter after scope reduction\?$/i.exec(body); + if (!choice) return false; + const label = (s: string) => s.trim().replace(/\s*\(recommended\)$/i, ''); + const injected = q.options.find(o => /^Constructor injection$/i.test(label(o.label))); + const imported = q.options.find(o => /^Module-level import$/i.test(label(o.label))); + if (!injected || !imported) return false; + // Consume both descriptions: the current-plan coupling and the offered + // alternative must be affirmative, not quoted, conditional, or mixed with new work. + const core = (s: string) => normalize(s).replace(/ Completeness: (?:10|[0-9])\/10\. \(human: (?:no change|~?\d+(?:\.\d+)?(?:min|h| days?)) \/ CC: (?:no change|~?\d+(?:\.\d+)?(?:min|h))\)$/, ''); + return core(injected.description ?? '') === `${choice[1]} receives the cache adapter as a constructor argument (or factory function parameter). Tests pass a stub; production passes the real adapter. Eliminates module-level mutable state entirely. Requires wiring at the call site.` && + core(imported.description ?? '') === `${choice[1]} imports the adapter directly at module scope, same pattern as the current plan. Works fine in production; makes tests require module-level mocking (jest.mock, proxyquire). Matches the existing codebase pattern if that's what's already used.`; +} + +/** A direct shared-state risk is a finding even when its qid omits the section name. */ +function engSharedMutableCacheAUQ(fp: AskUserQuestionFingerprint): boolean { + const call = fp.nativeCall; + if (call?.answered !== true || call.failed !== false || call.questions.length !== 1 || + !Array.isArray(call.unansweredQuestionIndices) || call.unansweredQuestionIndices.length || + fp.signature !== `${call.sessionId}:${call.toolUseId}`) return false; + const q = call.questions[0]!; + if (q.multiSelect || !/^Shared cache$/i.test(q.header.trim()) || q.options.length !== 3 || + new Set(q.options.map(o => o.label)).size !== 3 || + q.options.filter(o => o.label === call.answers?.[q.question]).length !== 1 || + fp.options.length !== 3 || !fp.options.every((o, i) => o.index === i + 1 && o.label === q.options[i]!.label)) return false; + const ids = [...q.question.matchAll(/]+)>/gi)]; + if (ids.length !== 1 || (q.question.match(/ s.trim().replace(/\s+/g, ' '); + const body = normalize(q.question.replace(/\s*]+>\s*$/i, '')); + const risk = /^D\s*\d+\s*[—–:-]\s*Architecture: Two services share a global mutable ([A-Za-z_$][\w$]*) via module-level export\. This is the #[1-9]\d* reliability risk in multi-tenant auth [—–-] concurrent mutations can corrupt tenant isolation\. How should the plan address this\?$/i.exec(body); + if (!risk) return false; + const label = (s: string) => s.trim().replace(/\s*\(recommended\)$/i, ''); + const injected = q.options.find(o => /^Dependency injection$/i.test(label(o.label))); + const guarded = q.options.find(o => /^Mutation guards on the global$/i.test(label(o.label))); + const accepted = q.options.find(o => /^Accept as-is, flag as known risk$/i.test(label(o.label))); + if (!injected || !guarded || !accepted) return false; + const core = (s: string) => normalize(s).replace(/ Completeness: (?:10|[0-9])\/10\.$/, ''); + const remedy = /^The plan is updated to pass ([A-Za-z_$][\w$]*) as a constructor argument to both ([A-Za-z_$][\w$]*) and ([A-Za-z_$][\w$]*)\. No module-level mutable export\. Tests can inject a mock\/stub\. Single shared instance still possible at the app root\.$/.exec(core(injected.description ?? '')); + return Boolean(remedy && remedy[1] === risk[1] && remedy[2] !== remedy[3] && + core(guarded.description ?? '') === 'Keep the global export but wrap every mutation site in explicit locking or compare-and-swap. Safer than bare shared state but still couples both services to the global. Adds concurrency primitives that need their own tests.' && + core(accepted.description ?? '') === 'Note the shared-global pattern in the review report as a known risk. Leave the plan unchanged. Suitable only if the runtime is single-threaded and concurrent mutation is architecturally impossible.'); +} + /** An answered substantive finding can start review even in a mixed setup packet. */ export const engFirstReviewAUQ: Step0BoundaryPredicate = (fp) => { - if (engExplicitRepairAUQ(fp) || engArchitectureChoiceAUQ(fp)) return true; + if (engExplicitRepairAUQ(fp) || engArchitectureChoiceAUQ(fp) || engDependencyBindingAUQ(fp) || engSharedMutableCacheAUQ(fp)) return true; const call = fp.nativeCall; if (!call?.answered || call.failed) return false; return call.questions.some(q => { diff --git a/test/helpers/design-count-review.ts b/test/helpers/design-count-review.ts index db077b295..65bf571a3 100644 --- a/test/helpers/design-count-review.ts +++ b/test/helpers/design-count-review.ts @@ -29,11 +29,37 @@ export function isDesignCountSetup(fp: AskUserQuestionFingerprint): boolean { q.options.some(option => option.label === call.answers?.[q.question]); } +/** A numbered design-system amendment can be the first review decision. */ +function numberedVisualHierarchyFinding(fp: AskUserQuestionFingerprint): boolean { + const call = fp.nativeCall; + if (!call || call.answered !== true || call.failed !== false || !call.sessionId || !call.toolUseId || + call.questions.length !== 1 || !Array.isArray(call.unansweredQuestionIndices) || call.unansweredQuestionIndices.length || + fp.signature !== `${call.sessionId}:${call.toolUseId}` || + (fp.nativeQuestionIndex !== undefined && fp.nativeQuestionIndex !== 0)) return false; + const q = call.questions[0]!; + const finding = /^Gap ([1-9]\d*) of ([1-9]\d*)\s*[—–-]\s*([A-Za-z][A-Za-z0-9_-]{0,39}) button visual hierarchy: apply DESIGN\.md primary button style\?$/i.exec(q.question.trim()); + if (!finding || Number(finding[1]) > Number(finding[2]) || + !new RegExp(`^Gap ${finding[1]}: Button$`, 'i').test(q.header.trim()) || q.multiSelect || q.options.length !== 2 || + fp.options.length !== 2 || !fp.options.every((o, i) => o.index === i + 1 && o.label === q.options[i]!.label) || + !q.options.some(o => o.label === call.answers?.[q.question])) return false; + const labels = q.options.map(o => o.label.trim().replace(/\s*\(recommended\)\s*$/i, '')); + const apply = labels.findIndex(s => /^Apply DESIGN\.md fix$/i.test(s)); + const defer = labels.findIndex(s => /^Defer to implementation$/i.test(s)); + if (apply < 0 || defer < 0 || apply === defer) return false; + const control = '[A-Za-z][A-Za-z0-9_-]{0,39}'; + const amendment = new RegExp(`^Add to plan: ${finding[3]} gets #[0-9a-f]{6} filled \\+ (?:white|black) text \\(primary\\); ${control}(?:, ${control})*(?:,? and ${control})? get neutral ghost style\\. Closes the visual hierarchy gap exactly as DESIGN\\.md specifies\\. Implementation task T[1-9]\\d* becomes committed\\.$`, 'i'); + // Both offered bodies describe the actual style amendment or its deferral; + // readiness, a source-selection question, or an example is not this finding. + return amendment.test(q.options[apply]!.description?.trim() ?? '') && + /^Leave the gap named but unresolved\. Engineer decides the button styles at implementation time without a spec\. Risk: inconsistency with the design system or re-work after review\.$/i.test(q.options[defer]!.description?.trim() ?? ''); +} + /** A completed finding can start the passes when the caller already supplied the focus. */ export function isDesignCountFirstReview(fp: AskUserQuestionFingerprint): boolean { const call = fp.nativeCall; if (!call?.answered || call.failed) return false; if (isDesignCountSetup(fp)) return false; + if (numberedVisualHierarchyFinding(fp)) return true; if (designFirstReviewAUQ(fp)) return true; return call.questions.some(q => { if (!call.answers?.[q.question] || q.options.length < 2) return false; diff --git a/test/helpers/devex-count-fixture.ts b/test/helpers/devex-count-fixture.ts index 537464f3a..1f4ad2827 100644 --- a/test/helpers/devex-count-fixture.ts +++ b/test/helpers/devex-count-fixture.ts @@ -355,6 +355,21 @@ function answeredDocumentationFollowup(fp: AskUserQuestionFingerprint): boolean if (selected.length !== 1 || ids.length !== 1 || (q.question.match(/]+>/i, '').trim(); + if (/^plan-devex-review-todo\d+-migration-guide$/i.test(ids[0]![1]!) && /^TODO[- ]\d+ Migration$/i.test(q.header.trim())) { + // A written upgrade guide is additional work beyond the accepted runtime + // compatibility shim. Require that distinct gap and the selected doc task; + // a recap, hypothetical example or unselected guide cannot supply it. + const parts = q.question.replace(/\s*]+>\s*$/i, '').trim().split(/\n\s*\n/); + const compact = (text: string | undefined) => (text ?? '').replace(/\s+/g, ' ').trim(); + return call.answered === true && parts.length === 5 && + /^D\s*\d+\s*[—–:-]\s*TODO: should the plan include a v\d+→v\d+ written migration guide\?$/i.test(compact(parts[0])) && + /^The deprecation shim \(T\d+\) handles the runtime experience: v\d+ callers get a DeprecationWarning naming `[a-z_]\w*\(\)` as the replacement\. But there is currently no written migration guide in docs\/\.$/i.test(compact(parts[1])) && + /^A one-page migration guide covers: - What changed \(`[a-z_]\w*\(\)` → `[a-z_]\w*\(\)`\) - What stayed the same \(all other APIs\) - How to find and update callsites \(grep for `[^`\n]+`\) - When the shim is removed \(e\.g\., v\d+(?:\.\d+)?\)$/i.test(compact(parts[2])) && + /^Without it, developers upgrading a large codebase need to discover the change at each call site rather than planning the migration upfront\. The changelog has the what; the guide provides the how and the timeline\.$/i.test(compact(parts[3])) && + /^Completeness: A=(?:10|[0-9])\/10 \(complete\), B=(?:10|[0-9])\/10 \(runtime-only, no planning\), C=(?:10|[0-9])\/10$/i.test(compact(parts[4])) && + /^Add to TODOS\.md [—–-] include migration guide in plan$/i.test(label) && + /^Add docs\/migration-v\d+-v\d+\.md as a P[0-3] task\. One page covering the rename, unchanged APIs, grep command to find callsites, and shim removal timeline\. Completeness: (?:10|[0-9])\/10\.$/i.test(compact(selected[0]!.description)); + } if (ids[0]![1] === 'devex-api-key-docs' && /^API key docs$/i.test(q.header.trim())) { // The earlier auth-error decision changes runtime diagnostics. This one // adds the missing acquisition instructions to the README itself. diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 63007a454..cab45aabc 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -100,17 +100,17 @@ export const E2E_TOUCHFILES: Record = { // include question-tuning.ts and generate-ask-user-format.ts because the // AUTO_DECIDE preamble injection lives there and changes can flip the // regression test outcome between 'asked' and 'auto_decided'. - 'plan-ceo-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'], - 'plan-eng-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'], - 'plan-design-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts', 'test/skill-e2e-design.test.ts'], - 'plan-devex-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts'], + 'plan-ceo-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'], + 'plan-eng-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'], + 'plan-design-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts', 'test/skill-e2e-design.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'], + 'plan-devex-review-plan-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'], // Covers ceo (preamble misfire) + eng/design (scope-gate bypass must not // fire outside plan mode) + the named-target exception case. 4 PTY runs; // in CI these run CONCURRENT with the rest of the pty-plan-smoke suite // (--max-concurrency + --retry 1), so worst-case cost is ~2x a single // pass of each, sharing the API budget with sibling tests — not the // sequential ~+10min a local read suggests. - 'plan-mode-no-op': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-mode-no-op.test.ts'], + 'plan-mode-no-op': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-mode-no-op.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'], // v1.21+ AskUserQuestion-blocked regression tests — Conductor launches // claude with `--disallowedTools AskUserQuestion --permission-mode default` @@ -120,7 +120,7 @@ export const E2E_TOUCHFILES: Record = { // INSIDE the existing 4 plan-X-review-plan-mode test files (covered // transitively by the entries above). Two new standalone files exist for // skills with no prior plan-mode test: - 'office-hours-auto-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-office-hours-auto-mode.test.ts'], + 'office-hours-auto-mode': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-office-hours-auto-mode.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'], 'office-hours-phase4-fork': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'office-hours/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/question-tuning.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours-phase4.test.ts'], 'llm-judge-recommendation': ['codex/**', 'test/helpers/llm-judge.ts', 'test/llm-judge-recommendation.test.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'codex/SKILL.md.tmpl', 'scripts/resolvers/review.ts'], // v1.21+ AUTO_DECIDE preserve eval (periodic). Verifies the Tool resolution @@ -128,20 +128,20 @@ export const E2E_TOUCHFILES: Record = { // written a never-ask preference, AUQ should still auto-decide rather than // surfacing the question. Touches the question-tuning + preference // infrastructure plus the resolvers that own the AUTO_DECIDE preamble. - 'auto-decide-preserved': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-auto-decide-preserved.test.ts'], + 'auto-decide-preserved': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'plan-ceo-review/**', 'bin/gstack-question-preference', 'bin/gstack-config', 'bin/gstack-slug', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-auto-decide-preserved.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'], // Conductor → prose decision brief (Conductor signal makes prose the default; // the PreToolUse hook denies the flaky tool). Touches the resolver that owns // the Conductor rule, the preamble signal, the hook, and the detection helper. - 'conductor-prose': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble.ts', 'plan-eng-review/**', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-conductor-prose.test.ts'], + 'conductor-prose': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble.ts', 'plan-eng-review/**', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-conductor-prose.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'], // Real-PTY E2E batch (#6 new tests on the harness). // Each one tests behavior the SDK harness can't observe (rendered TTY, // numbered-option lists, multi-phase ordering, idempotency state echo). 'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-ask-user-question-format-compliance.test.ts'], - 'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/ceo-mode-option.ts', 'test/ceo-mode-option.test.ts', 'test/pty-option-selection.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-ceo-mode-routing.test.ts', 'test/ceo-mode-posture-native.test.ts', 'test/fixtures/ceo-hold-posture-l.json', 'test/ceo-mode-labels-native.test.ts', 'test/fixtures/ceo-mode-labels-l.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/ceo-mode-prerequisite.test.ts', 'test/fixtures/ceo-mode-prerequisite-o-calls.json', 'test/fixtures/ceo-mode-prerequisite-q-call.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/ceo-posture-packet.test.ts', 'test/fixtures/design-preview-v-screen.txt'], - 'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'], - 'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-ship-idempotency.test.ts'], + 'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/ceo-mode-option.ts', 'test/ceo-mode-option.test.ts', 'test/pty-option-selection.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-ceo-mode-routing.test.ts', 'test/ceo-mode-posture-native.test.ts', 'test/fixtures/ceo-hold-posture-l.json', 'test/ceo-mode-labels-native.test.ts', 'test/fixtures/ceo-mode-labels-l.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/ceo-mode-prerequisite.test.ts', 'test/fixtures/ceo-mode-prerequisite-o-calls.json', 'test/fixtures/ceo-mode-prerequisite-q-call.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/ceo-posture-packet.test.ts', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'], + 'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-with-ui.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'], + 'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-ship-idempotency.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'], 'tpa-present': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'], 'tpa-absent-linux': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'], 'tpa-broken': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'], @@ -154,34 +154,34 @@ export const E2E_TOUCHFILES: Record = { // the selector sets GSTACK_CARVE_SKILL= to scope cost to the changed // skill (D-CODEX A). Touching the registry/helper or sections.ts runs all. 'carve-section-loading': ['scripts/resolvers/composition.ts', 'test/autoplan-review-discovery.test.ts', 'test/autoplan-phase-order.test.ts', 'design-html/**', 'design-shotgun/**', 'qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/resolvers/redact-doc.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/session-runner-tools.test.ts', 'test/carve-section-loading.test.ts', 'bin/gstack-autoplan-snapshot.ts', 'test/autoplan-snapshot.test.ts', 'test/autoplan-init.test.ts', 'test/autoplan-obligations.test.ts', 'test/fixtures/autoplan/t-ceo-omitted-obligations.json', 'test/fixtures/autoplan/u-ceo-original-loss.json', 'test/fixtures/autoplan/v-ceo-dangling-references.json'], - 'autoplan-chain-pty': ['scripts/resolvers/composition.ts', 'test/autoplan-review-discovery.test.ts', 'test/autoplan-phase-order.test.ts', 'autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/autoplan-dashboard.md', 'test/autoplan-chain-fixture.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'bin/gstack-config', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/autoplan-setup-question.ts', 'test/autoplan-setup-question.test.ts', 'test/helpers/autoplan-phase-observer.ts', 'test/autoplan-phase-observer.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/skill-e2e-autoplan-chain.test.ts', 'bin/gstack-autoplan-snapshot.ts', 'test/autoplan-snapshot.test.ts', 'test/autoplan-init.test.ts', 'test/autoplan-obligations.test.ts', 'test/fixtures/autoplan/t-ceo-omitted-obligations.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/helpers/eval-budgets.ts', 'test/autoplan-eval-budget.test.ts', 'test/eval-budgets-policy.test.ts', 'test/eval-detach-timeout-floor.test.ts', 'scripts/test-paid-shards.ts', '.github/workflows/evals-periodic.yml', 'test/fixtures/autoplan-routing-n-screen.txt', 'test/autoplan-routing-o.test.ts', 'test/fixtures/autoplan-routing-o-screen.txt', 'test/autoplan-setup-packet-o.test.ts', 'test/fixtures/autoplan-setup-packet-o-screen.txt', 'test/fixtures/autoplan-setup-packet-o-call.json', 'test/fixtures/autoplan/u-ceo-original-loss.json', 'test/fixtures/autoplan/v-ceo-dangling-references.json'], + 'autoplan-chain-pty': ['scripts/resolvers/composition.ts', 'test/autoplan-review-discovery.test.ts', 'test/autoplan-phase-order.test.ts', 'autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/autoplan-dashboard.md', 'test/autoplan-chain-fixture.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'bin/gstack-config', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/autoplan-setup-question.ts', 'test/autoplan-setup-question.test.ts', 'test/helpers/autoplan-phase-observer.ts', 'test/autoplan-phase-observer.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/skill-e2e-autoplan-chain.test.ts', 'bin/gstack-autoplan-snapshot.ts', 'test/autoplan-snapshot.test.ts', 'test/autoplan-init.test.ts', 'test/autoplan-obligations.test.ts', 'test/fixtures/autoplan/t-ceo-omitted-obligations.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/helpers/eval-budgets.ts', 'test/autoplan-eval-budget.test.ts', 'test/eval-budgets-policy.test.ts', 'test/eval-detach-timeout-floor.test.ts', 'scripts/test-paid-shards.ts', '.github/workflows/evals-periodic.yml', 'test/fixtures/autoplan-routing-n-screen.txt', 'test/autoplan-routing-o.test.ts', 'test/fixtures/autoplan-routing-o-screen.txt', 'test/autoplan-setup-packet-o.test.ts', 'test/fixtures/autoplan-setup-packet-o-screen.txt', 'test/fixtures/autoplan-setup-packet-o-call.json', 'test/fixtures/autoplan/u-ceo-original-loss.json', 'test/fixtures/autoplan/v-ceo-dangling-references.json', 'test/fixtures/autoplan-setup-z-packet.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'], // Per-finding AskUserQuestion count + review-report-at-bottom assertion. // Each test drives its skill end-to-end; touchfiles include preamble + // completion-status resolvers because they affect question cadence and // terminal output (the regression surface this test catches). - 'plan-ceo-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'test/skill-ceo-section-ordering.test.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/ceo-completion-handoff.ts', 'test/ceo-completion-handoff.test.ts', 'test/ceo-count-s-terminals.test.ts', 'test/fixtures/ceo-count-s-paired.json', 'test/fixtures/ceo-completion-handoff-calls.json', 'test/fixtures/ceo-completion-handoff-j-calls.json', 'test/fixtures/ceo-completion-handoff-k-calls.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-ceo-finding-count.test.ts', 'test/ceo-completion-handoff-l.test.ts', 'test/fixtures/ceo-completion-handoff-l-calls.json', 'test/ceo-completion-handoff-m.test.ts', 'test/fixtures/ceo-completion-handoff-m-call.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/ceo-handoff-n-calls.json', 'test/ceo-completion-handoff-o.test.ts', 'test/fixtures/ceo-completion-handoff-o-call.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/helpers/ceo-approach-pick.ts', 'test/ceo-approach-pick.test.ts', 'test/fixtures/ceo-approach-q-call.json', 'test/fixtures/ceo-approach-r-call.json', 'test/fixtures/ceo-approach-r-distinct-call.json', 'test/fixtures/ceo-approach-q-paired-call.json', 'test/fixtures/ceo-completion-handoff-q-call.json', 'test/fixtures/ceo-completion-handoff-r-calls.json', 'test/fixtures/ceo-completion-handoff-t-call.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/ceo-completion-handoff-u-call.json', 'test/fixtures/ceo-completion-handoff-v-call.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/plan-count-ceo-body-finding.test.ts', 'test/fixtures/ceo-count-w-paired.json', 'test/fixtures/ceo-completion-handoff-w-call.json', 'test/fixtures/ceo-approach-y-call.json', 'test/fixtures/ceo-approach-y-screen.txt', 'test/ceo-handoff-y.test.ts', 'test/fixtures/ceo-handoff-y-call.json'], - 'plan-eng-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-eng-finding-count.test.ts', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/eng-first-review-t.test.ts', 'test/fixtures/eng-batching-t-calls.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/eng-scope-y.test.ts', 'test/fixtures/eng-scope-y-calls.json'], - 'plan-design-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/design-count-review.ts', 'test/design-count-review.test.ts', 'test/fixtures/design-review-j-calls.json', 'test/helpers/design-count-outside.ts', 'test/design-count-outside.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts', 'test/fixtures/design-review-l-calls.json', 'test/design-completion-handoff.test.ts', 'test/fixtures/design-handoff-l-calls.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/design-review-n-calls.json', 'test/design-completion-handoff-scored.test.ts', 'test/fixtures/design-handoff-n-calls.json', 'test/fixtures/design-handoff-q-calls.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/design-completion-handoff-u.test.ts', 'test/fixtures/design-handoff-u-calls.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/helpers/design-artifact-question.ts', 'test/design-artifact-question.test.ts', 'test/fixtures/design-artifacts-w-calls.json', 'test/fixtures/design-outside-y-calls.json', 'test/fixtures/design-boundaries-y-calls.json'], - 'plan-devex-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/devex-count-fixture.ts', 'test/devex-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts', 'test/fixtures/dx-prerequisite-r-call.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/devex-review-n-calls.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/devex-output-o.test.ts', 'test/fixtures/devex-review-o-calls.json', 'test/fixtures/devex-output-o-retry-call.json', 'test/devex-setup-remedy-o.test.ts', 'test/fixtures/devex-review-o-retry-calls.json', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/fixtures/devex-review-t-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/devex-count-u-calls.json', 'test/fixtures/devex-count-u-retry-calls.json', 'test/fixtures/devex-empathy-v-calls.json', 'test/fixtures/devex-handoff-v-call.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/fixtures/devex-count-y-calls.json'], + 'plan-ceo-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'test/skill-ceo-section-ordering.test.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/ceo-completion-handoff.ts', 'test/ceo-completion-handoff.test.ts', 'test/ceo-count-s-terminals.test.ts', 'test/fixtures/ceo-count-s-paired.json', 'test/fixtures/ceo-completion-handoff-calls.json', 'test/fixtures/ceo-completion-handoff-j-calls.json', 'test/fixtures/ceo-completion-handoff-k-calls.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-ceo-finding-count.test.ts', 'test/ceo-completion-handoff-l.test.ts', 'test/fixtures/ceo-completion-handoff-l-calls.json', 'test/ceo-completion-handoff-m.test.ts', 'test/fixtures/ceo-completion-handoff-m-call.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/ceo-handoff-n-calls.json', 'test/ceo-completion-handoff-o.test.ts', 'test/fixtures/ceo-completion-handoff-o-call.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/helpers/ceo-approach-pick.ts', 'test/ceo-approach-pick.test.ts', 'test/fixtures/ceo-approach-q-call.json', 'test/fixtures/ceo-approach-r-call.json', 'test/fixtures/ceo-approach-r-distinct-call.json', 'test/fixtures/ceo-approach-q-paired-call.json', 'test/fixtures/ceo-completion-handoff-q-call.json', 'test/fixtures/ceo-completion-handoff-r-calls.json', 'test/fixtures/ceo-completion-handoff-t-call.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/ceo-completion-handoff-u-call.json', 'test/fixtures/ceo-completion-handoff-v-call.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/plan-count-ceo-body-finding.test.ts', 'test/fixtures/ceo-count-w-paired.json', 'test/fixtures/ceo-completion-handoff-w-call.json', 'test/fixtures/ceo-approach-y-call.json', 'test/fixtures/ceo-approach-y-screen.txt', 'test/ceo-handoff-y.test.ts', 'test/fixtures/ceo-handoff-y-call.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/ceo-handoff-z-call.json'], + 'plan-eng-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-eng-finding-count.test.ts', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/eng-first-review-t.test.ts', 'test/fixtures/eng-batching-t-calls.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/eng-scope-y.test.ts', 'test/fixtures/eng-scope-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/eng-binding-z.test.ts', 'test/fixtures/eng-binding-z-calls.json', 'test/eng-binding-retry-z.test.ts', 'test/fixtures/eng-binding-retry-z-calls.json'], + 'plan-design-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/design-count-review.ts', 'test/design-count-review.test.ts', 'test/fixtures/design-review-j-calls.json', 'test/helpers/design-count-outside.ts', 'test/design-count-outside.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts', 'test/fixtures/design-review-l-calls.json', 'test/design-completion-handoff.test.ts', 'test/fixtures/design-handoff-l-calls.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/design-review-n-calls.json', 'test/design-completion-handoff-scored.test.ts', 'test/fixtures/design-handoff-n-calls.json', 'test/fixtures/design-handoff-q-calls.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/design-completion-handoff-u.test.ts', 'test/fixtures/design-handoff-u-calls.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/helpers/design-artifact-question.ts', 'test/design-artifact-question.test.ts', 'test/fixtures/design-artifacts-w-calls.json', 'test/fixtures/design-outside-y-calls.json', 'test/fixtures/design-boundaries-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/design-gap-z-calls.json'], + 'plan-devex-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/devex-count-fixture.ts', 'test/devex-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts', 'test/fixtures/dx-prerequisite-r-call.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/devex-review-n-calls.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/devex-output-o.test.ts', 'test/fixtures/devex-review-o-calls.json', 'test/fixtures/devex-output-o-retry-call.json', 'test/devex-setup-remedy-o.test.ts', 'test/fixtures/devex-review-o-retry-calls.json', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/fixtures/devex-review-t-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/devex-count-u-calls.json', 'test/fixtures/devex-count-u-retry-calls.json', 'test/fixtures/devex-empathy-v-calls.json', 'test/fixtures/devex-handoff-v-call.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/fixtures/devex-count-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/devex-count-z-calls.json', 'test/fixtures/devex-handoff-z-call.json'], // Gate-tier reviewCount-floor counterparts. Catch the May 2026 transcript // bug (model wrote a plan-mode plan and ExitPlanMode'd without firing any // review-phase AskUserQuestion). Uses runPlanSkillFloorCheck — minimal // "did agent fire ANY AUQ?" observer that exits early on first non-permission // numbered-option render. ~1-3 min typical wall time per test, ~$2-6 total. - 'plan-eng-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-finding-floor.test.ts', 'test/eng-first-review-t.test.ts', 'test/fixtures/eng-batching-t-calls.json', 'test/eng-scope-y.test.ts', 'test/fixtures/eng-scope-y-calls.json'], - 'plan-ceo-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-finding-floor.test.ts'], - 'plan-design-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-design-finding-floor.test.ts'], - 'plan-devex-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-devex-finding-floor.test.ts'], + 'plan-eng-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-finding-floor.test.ts', 'test/eng-first-review-t.test.ts', 'test/fixtures/eng-batching-t-calls.json', 'test/eng-scope-y.test.ts', 'test/fixtures/eng-scope-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/eng-binding-z.test.ts', 'test/fixtures/eng-binding-z-calls.json', 'test/eng-binding-retry-z.test.ts', 'test/fixtures/eng-binding-retry-z-calls.json'], + 'plan-ceo-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-finding-floor.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'], + 'plan-design-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-design-finding-floor.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'], + 'plan-devex-finding-floor': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-devex-finding-floor.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'], // Multi-finding batching regression — periodic tier complement to the // gate-tier finding-floor. Catches the May 2026 transcript shape where // a model fires one AUQ then batches the rest into a "## Decisions to // confirm" plan write. runPlanSkillFloorCheck cannot detect that shape // (it exits on first AUQ); runPlanSkillCounting can. - 'plan-eng-multi-finding-batching': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/eng-first-review-t.test.ts', 'test/fixtures/eng-batching-t-calls.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/eng-scope-y.test.ts', 'test/fixtures/eng-scope-y-calls.json'], - 'plan-ceo-split-overflow': ['bin/gstack-config', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json'], + 'plan-eng-multi-finding-batching': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/eng-first-review-t.test.ts', 'test/fixtures/eng-batching-t-calls.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/eng-scope-y.test.ts', 'test/fixtures/eng-scope-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/eng-binding-z.test.ts', 'test/fixtures/eng-binding-z-calls.json', 'test/eng-binding-retry-z.test.ts', 'test/fixtures/eng-binding-retry-z-calls.json'], + 'plan-ceo-split-overflow': ['bin/gstack-config', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'], 'brain-privacy-gate': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-brain-privacy-gate.test.ts'], // /setup-gbrain Path 4 (Remote MCP) — happy + bad-token end-to-end via diff --git a/test/plan-count-dx-handoff.test.ts b/test/plan-count-dx-handoff.test.ts index 17dc5cb96..df8947fa7 100644 --- a/test/plan-count-dx-handoff.test.ts +++ b/test/plan-count-dx-handoff.test.ts @@ -144,3 +144,60 @@ describe('completed DX handoff with a result recap after its navigation question } finally { f.cleanup(); } }); }); + +// Actual Z native calls, owned Exit hook, report bytes and original file mtime. +import zCaptured from './fixtures/devex-handoff-z-call.json'; +describe('Z completed DX navigation over specified tasks', () => { + function run(edit?: (calls: NativePlanQuestionCall[], hook: any) => void, reportTime = zCaptured.provenance.reportMtimeMs, noHook = false) { + const dir=fs.mkdtempSync(path.join(os.tmpdir(),'dx-z-handoff-')); + try { + const report=path.join(dir,'report.md'), pending=path.join(dir,'pending.json'); + fs.writeFileSync(report,zCaptured.reportContent);fs.utimesSync(report,reportTime/1000,reportTime/1000); + const calls=structuredClone(zCaptured.calls) as NativePlanQuestionCall[],hook=structuredClone(zCaptured.pendingExit);edit?.(calls,hook); + fs.writeFileSync(pending,JSON.stringify(hook)); + const started=Date.parse(zCaptured.provenance.startedAt),raw={status:'ready' as const,calls,assistantMessages:[]}; + const observed=noHook?raw:withPendingExit(raw,pending,zCaptured.provenance.capture.cwd,zCaptured.provenance.capture.claudeConfigDir,started,zCaptured.screen); + return hasNativePlanTerminal(observed,report,started,'plan_ready'); + }finally{fs.rmSync(dir,{recursive:true,force:true});} + } + const change=(c:NativePlanQuestionCall,from:string,to:string)=>{const q=c.questions[0]!;const answer=c.answers![q.question]!;q.question=q.question.replace(from,to);c.answers={[q.question]:answer};}; + test('exact native navigation preserves the original report after all issue answers',()=>{ + expect(zCaptured.calls).toHaveLength(9); + expect(Date.parse(zCaptured.calls.at(-2)!.answeredAt)).toBeLessThan(zCaptured.provenance.reportMtimeMs); + expect(Date.parse(zCaptured.calls.at(-1)!.answeredAt)).toBeGreaterThan(zCaptured.provenance.reportMtimeMs); + expect(zCaptured.calls.at(-1)!.answers[zCaptured.calls.at(-1)!.questions[0]!.question]).toBe('Run /plan-eng-review next (Recommended)'); + expect(run()).toBe(true);expect(run(calls=>calls.at(-1)!.questions[0]!.options.reverse())).toBe(true); + for(const index of [1,2])expect(run(calls=>{const c=calls.at(-1)!,q=c.questions[0]!;c.answers={[q.question]:q.options[index]!.label};})).toBe(true); + }); + test('whole closed body and all descriptions cannot add unresolved product work',()=>{ + for(const [from,to] of [ + ['DX review complete.','DX review complete only after fixing errors.'],['7 tasks specified','7 tasks proposed'],['7 tasks specified','8 tasks specified'], + ["What's next?",'Should we add another migration guide?'],['Next steps','Issue decision'],['warrant an eng review','require another product decision'], + ['',''], + ])expect(run(calls=>{const c=calls.at(-1)!;if(from==='Next steps')c.questions[0]!.header=to!;else change(c,from!,to!);})).toBe(false); + for(const text of [' One task remains unresolved.',' Add a missing test.',' '])expect(run(calls=>change(calls.at(-1)!,'',''+text))).toBe(false); + for(const index of [0,1,2])expect(run(calls=>{calls.at(-1)!.questions[0]!.options[index]!.description+=' Also change authentication.';})).toBe(false); + for(const prefix of ['> ','```text\n','For example: '])expect(run(calls=>{const c=calls.at(-1)!;change(c,c.questions[0]!.question,prefix+c.questions[0]!.question);})).toBe(false); + expect(run(calls=>{calls.at(-1)!.questions[0]!.options[1]!.description='Start implementing a newly added feature.';})).toBe(false); + }); + test('native completion, exact offered answer and current owned Exit remain mandatory',()=>{ + for(const edit of [ + (c:NativePlanQuestionCall)=>{c.answered=false;},(c:NativePlanQuestionCall)=>{delete (c as any).answered;}, + (c:NativePlanQuestionCall)=>{c.failed=true;},(c:NativePlanQuestionCall)=>{delete c.failed;}, + (c:NativePlanQuestionCall)=>{c.unansweredQuestionIndices=[0];},(c:NativePlanQuestionCall)=>{delete c.unansweredQuestionIndices;}, + (c:NativePlanQuestionCall)=>{c.answers={[c.questions[0]!.question]:'Not offered'};}, + (c:NativePlanQuestionCall)=>{c.questions[0]!.multiSelect=true;},(c:NativePlanQuestionCall)=>{c.questions.push(structuredClone(c.questions[0]!));}, + (c:NativePlanQuestionCall)=>{c.questions[0]!.options.push(structuredClone(c.questions[0]!.options[0]!));}, + ])expect(run(calls=>edit(calls.at(-1)!))).toBe(false); + expect(run(undefined,undefined,true)).toBe(false); + expect(run((_calls,h)=>{h.sessionId='foreign';})).toBe(false); + expect(run((_calls,h)=>{h.cwd='/foreign';})).toBe(false); + expect(run((_calls,h)=>{h.timestamp='2026-09-09T11:00:00Z';})).toBe(false); + expect(run(calls=>{calls.splice(0,calls.length-1);})).toBe(false); + }); + test('report freshness remains bound to every substantive answer and task numbers are not fixture ids',()=>{ + expect(run(calls=>{calls.at(-2)!.answeredAt=new Date(zCaptured.provenance.reportMtimeMs+1).toISOString();})).toBe(false); + expect(run(undefined,Date.parse(zCaptured.calls.at(-2)!.answeredAt)-1)).toBe(false); + expect(run(calls=>{const c=calls.at(-1)!;for(const [from,to] of [['D9','D19'],['7 tasks','8 tasks'],['2 P2','3 P2']])change(c,from!,to!);c.questions[0]!.options[1]!.description=c.questions[0]!.options[1]!.description!.replace('T1–T7','T1–T8');})).toBe(true); + }); +}); diff --git a/test/plan-count-truncated-question.test.ts b/test/plan-count-truncated-question.test.ts new file mode 100644 index 000000000..9c5895c75 --- /dev/null +++ b/test/plan-count-truncated-question.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, test } from 'bun:test'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { capturePlanCountQuestion, matchesNativePlanQuestion, nativePlanCallFingerprint, planCountQuestionInput } from './helpers/claude-pty-runner'; +import { pickCeoCountQuestion } from './helpers/ceo-approach-pick'; +import { createPtyScreen } from './helpers/pty-screen'; +import type { NativePlanQuestionCall } from './helpers/plan-count-transcript'; +import completed from './fixtures/ceo-approach-z-call.json'; + +const screen = fs.readFileSync(path.join(import.meta.dir, 'fixtures/ceo-approach-z-screen.txt'), 'utf8'); +function pending(): NativePlanQuestionCall { + // No pending-only version survived the live capture. This is an explicit + // projection for replay, not evidence of metadata availability at input time. + const { answers, answeredAt, unansweredQuestionIndices, ...call } = structuredClone(completed); + return { ...call, answered: false, failed: false }; +} + +describe('native question truncated before its routing id', () => { + test('the actual complete 120x40 pane binds its exact native prefix and offered labels', async () => { + const viewport = await createPtyScreen(120, 40); + try { + viewport.write(screen.replaceAll('\n', '\r\n')); + const visible = await viewport.read(); + expect(visible.trimEnd()).toBe(screen.trimEnd()); + expect(visible).not.toContain(' { + for (const visible of [ + screen.replace('☐ Approach', '☐ Other'), + screen.replace('SQL injection bug', 'different defect'), + screen.replace('Dela…', 'Different…'), + screen.replace('Dela…', 'Dela'), + screen.replace('B — Proper Integration (recommended)', 'B — Different Integration (recommended)'), + screen.replace('3. C — Reject and Redesign', '3. Changed choice'), + screen.replace('4. Type something.', '4. Extra operation'), + screen.replace('↑/↓ to navigate', '↑/↓ to navigte'), + screen.replace(' · Esc to cancel', ''), + screen + '\nA later question?', + 'Quoted earlier menu:\n' + screen, + screen.slice(screen.indexOf('❯ 1.')), + screen.replace(/\n\n│[\s\S]*?\n\n❯/, '\n\n│ Which approach…\n\n❯'), + screen.replace('☐ Approach', '← ☐ Approach ☐ Other ✔ Submit →'), + ]) { + expect(visible).not.toBe(screen); + const call = pending(); + expect(matchesNativePlanQuestion(visible, call), visible).toBe(false); + expect(capturePlanCountQuestion(visible, new Set(), 0, true, call)?.nativeCall).toBeUndefined(); + } + const packet = pending(); packet.questions.push(structuredClone(packet.questions[0]!)); + expect(matchesNativePlanQuestion(screen, packet)).toBe(false); + }); + + test('answer state, failed calls and late metadata retain existing guards and deduplication', () => { + for (const call of [completed, { ...pending(), failed: true }]) { + expect(capturePlanCountQuestion(screen, new Set(), 0, true, call)?.nativeCall).toBeUndefined(); + } + const seen = new Set(); + const unbound = capturePlanCountQuestion(screen, seen, 0, true)!; + expect(unbound.nativeCall).toBeUndefined(); + expect(pickCeoCountQuestion(nativePlanCallFingerprint(pending(), 0, true), unbound)).toBeNull(); + expect(capturePlanCountQuestion(screen, seen, 1, true, pending())).toBeNull(); + const nativeSeen = new Set(); + expect(capturePlanCountQuestion(screen, nativeSeen, 0, true, pending())?.nativeCall).toBeDefined(); + expect(capturePlanCountQuestion(screen, nativeSeen, 1, true, pending())).toBeNull(); + expect(capturePlanCountQuestion(screen, nativeSeen, 2, true)).toBeNull(); + }); +}); + +test.skipIf(process.platform === 'win32')('real fake CLI receives the offered recommendation instead of default A', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'count-truncated-')); + const fake = path.join(dir, 'fake-claude'); + const worker = path.join(dir, 'worker.ts'); + const events = path.join(dir, 'events.jsonl'); + const output = path.join(dir, 'output.json'); + fs.writeFileSync(fake, `#!${process.execPath}\n` + String.raw` +import fs from 'node:fs';import path from 'node:path'; +const item=JSON.parse(process.env.TRUNCATED_CASE);const log=e=>fs.appendFileSync(item.events,JSON.stringify(e)+'\n'); +const sid='truncated-'+process.pid;const nativePath=path.join(process.env.CLAUDE_CONFIG_DIR,'projects',sid,sid+'.jsonl');fs.mkdirSync(path.dirname(nativePath),{recursive:true}); +const native=(role,content,extra={})=>fs.appendFileSync(nativePath,JSON.stringify({cwd:process.cwd(),sessionId:sid,isSidechain:false,timestamp:new Date().toISOString(),message:{role,content},...extra})+'\n'); +native('assistant',[{type:'text',text:'Fixture started.'}]);log({type:'start',pid:process.pid,cwd:process.cwd()}); +let stage='startup';process.stdin.setRawMode?.(true);process.stdin.on('data',data=>{ + const input=data.toString();log({type:'input',stage,input}); + if(stage==='startup'){ + stage='question';native('assistant',[{type:'tool_use',name:'AskUserQuestion',id:'truncated',input:{questions:[item.question]}}]); + process.stdout.write('\x1b[2J\x1b[H'+item.screen.replaceAll('\n','\r\n'));return; + } + if(stage!=='question'){log({type:'unexpected',input});return;} + const digit=/^[1-9]$/.exec(input)?.[0];if(!digit){log({type:'unexpected',input});return;} + stage='done';const choice=item.question.options[Number(digit)-1]?.label; + native('user',[{type:'tool_result',tool_use_id:'truncated',content:'Answered'}],{toolUseResult:{answers:{[item.question.question]:choice}}}); + log({type:'choice',choice,input}); + const q={header:'Finding',question:'Apply the repair?',options:[{label:'Fix'},{label:'Keep'}]}; + native('assistant',[{type:'tool_use',name:'AskUserQuestion',id:'finding',input:{questions:[q]}}]); + native('user',[{type:'tool_result',tool_use_id:'finding',content:'Answered'}],{toolUseResult:{answers:{[q.question]:'Fix'}}}); + process.stdout.write('\x1b[2J\x1b[HDone.\r\n'); +});process.on('SIGINT',()=>process.exit(0));process.stdin.resume(); +`); + fs.chmodSync(fake, 0o755); + const runner = pathToFileURL(path.join(import.meta.dir, 'helpers/claude-pty-runner.ts')).href; + const picker = pathToFileURL(path.join(import.meta.dir, 'helpers/ceo-approach-pick.ts')).href; + fs.writeFileSync(worker, `import {runPlanSkillCounting} from ${JSON.stringify(runner)};import {pickCeoCountQuestion} from ${JSON.stringify(picker)}; +const result=await runPlanSkillCounting({skillName:'plan-ceo-review',slashCommand:'/plan-ceo-review',followUpPrompt:'Review this fixture.',isLastStep0AUQ:()=>true,pickAUQ:pickCeoCountQuestion,defaultPick:1,reviewCountCeiling:1,timeoutMs:26000,env:{TRUNCATED_CASE:${JSON.stringify(JSON.stringify({ events, screen, question: completed.questions[0] }))}}});await Bun.write(${JSON.stringify(output)},JSON.stringify(result));`); + const child = Bun.spawn([process.execPath, worker], { env: { ...process.env, BROWSE_TERMINAL_BINARY: fake, EVALS_HERMETIC: '1' }, stdout: 'pipe', stderr: 'pipe' }); + const timer = setTimeout(() => child.kill('SIGKILL'), 30000); + try { + const [code, out, err] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]); + expect(code, out + err).toBe(0); + const result = JSON.parse(fs.readFileSync(output, 'utf8')); + expect(result.outcome, JSON.stringify(result)).toBe('ceiling_reached'); + expect(result.step0Count).toBe(1); + expect(result.reviewCount).toBe(1); + const rows = fs.readFileSync(events, 'utf8').trim().split('\n').map(line => JSON.parse(line)); + expect(rows.filter(row => row.type === 'input').map(row => row.input)).toEqual(['/plan-ceo-review\r', '2']); + expect(rows.find(row => row.type === 'choice').choice).toBe(completed.questions[0]!.options[1]!.label); + expect(rows.some(row => row.type === 'unexpected')).toBe(false); + expect(() => process.kill(rows[0].pid, 0)).toThrow(); + expect(fs.existsSync(rows[0].cwd)).toBe(false); + } finally { + clearTimeout(timer); child.kill('SIGKILL'); await child.exited; + if (fs.existsSync(events)) { + const first = JSON.parse(fs.readFileSync(events, 'utf8').split('\n')[0]!); + try { + const argv = process.platform === 'linux' + ? fs.readFileSync('/proc/' + first.pid + '/cmdline', 'utf8').split('\0') + : Bun.spawnSync(['ps', '-p', String(first.pid), '-o', 'command='], { timeout: 1000 }).stdout.toString().trim().split(/\s+/); + if (argv.includes(fake)) process.kill(first.pid, 'SIGKILL'); + } catch { /* owned child already closed */ } + } + fs.rmSync(dir, { recursive: true, force: true }); + } +}, 35000); diff --git a/test/session-runner-tools.test.ts b/test/session-runner-tools.test.ts index 9c2ca0fdc..dac2ea818 100644 --- a/test/session-runner-tools.test.ts +++ b/test/session-runner-tools.test.ts @@ -397,6 +397,8 @@ describe.skipIf(process.platform === 'win32')('session-runner explicit tool avai expect(child.prompt).toContain('all 11 sections an explicit outcome (including no issues or justified skips)'); expect(child.prompt).toContain('complete required registries, applicable diagrams, tasks, completion summary, and exact GSTACK REVIEW REPORT table'); expect(child.prompt).toContain('Cross-reference those records instead of repeating findings, option deliberations, diagrams, or registries'); + expect(child.prompt).toContain('Use compact outcome entries and short table cells; execute the review checklists without copying their questions or narrating every check into the artifact.'); + expect(child.prompt).toContain('Brevity must preserve every finding, accepted requirement, required field, and required diagram in its specified format.'); expect(child.prompt).toContain('unless that code is needed to specify an accepted plan change'); expect(child.prompt).toContain('execute the full review, perform every required lazy-file Read, and complete all required artifacts'); expect(child.prompt).toContain('MUST actually Read that sections/ file with the Read tool BEFORE doing the work it covers'); diff --git a/test/skill-e2e-plan-design-finding-count.test.ts b/test/skill-e2e-plan-design-finding-count.test.ts index b7e58d76e..1f8bebffd 100644 --- a/test/skill-e2e-plan-design-finding-count.test.ts +++ b/test/skill-e2e-plan-design-finding-count.test.ts @@ -34,6 +34,8 @@ const designSystem = [ '', 'Reuse the existing single-column settings shell: persistent app navigation,', 'page title and description, action group, then Profile and Notifications fieldsets.', + 'The existing description is “Manage your display name, email address, and notification preferences.”', + 'Preserve that description verbatim.', 'The form has a 640px maximum width. Reuse existing Button, Field, InlineStatus,', 'ErrorSummary and ConfirmationDialog components; no new component family is needed.', 'Profile contains Display name (text) and Email (email). Notifications contains', @@ -64,6 +66,8 @@ const designSystem = [ '', 'The established pending-action pattern is an inline spinner beside “Saving…”', 'inside the disabled Save button, aria-busy=true, with reduced-motion support.', + 'Save and Export are mutually exclusive: disable both while either is pending.', + 'After Save finishes, Export downloads the latest successfully saved preferences.', 'Success uses the persistent inline status “Saved” plus the save time (aria-live=polite).', 'The existing formatter renders “Saved at HH:mm” in the user’s local 24-hour time.', 'Editing a field away from its saved value changes that same persistent status', @@ -81,6 +85,7 @@ const designSystem = [ 'Validation errors appear beside fields and in a linked error summary; focus the', 'first invalid field. Network errors preserve edits and show Retry in that status area.', 'Retry is a sibling button beside the status text, outside its live region.', + 'Visible text stays “Retry”; its aria-label is “Retry save” or “Retry export” for that operation.', 'Export failures use that same error/retry area without changing unsaved fields;', 'Retry repeats Export. A successful download clears the Export error only.', 'At 640px and below, the three secondary actions share one equal-column row', @@ -100,6 +105,8 @@ const planDesign5Findings = (planPath: string) => [ 'The shell and components already exist. Profile and Notifications are the only', 'sections, with visible headings and associated field labels. The page header', 'contains the title, a short description, and Save/Reset/Cancel/Export actions.', + 'The existing description is “Manage your display name, email address, and notification preferences.”', + 'Preserve that description verbatim.', 'Preserve the approved single-column structure and component behavior.', 'The page title “Account settings” is h1. Profile and Notifications are h2', 'headings that label their fieldsets via aria-labelledby; no heading level is skipped.', @@ -129,12 +136,15 @@ const planDesign5Findings = (planPath: string) => [ 'Save is atomic: all fields persist together or none do, so partial success is', 'not exposed. Field validation, network failure, and successful-save feedback', 'use the exact existing DESIGN.md patterns. Preserve unsaved values after errors.', - 'Disable repeat Save submissions while pending. Reset restores saved values only', + 'Disable repeat Save submissions while pending. Save and Export are mutually', + 'exclusive: disable both while either is pending. After Save finishes, Export', + 'downloads the latest successfully saved preferences. Reset restores saved values only', 'after confirmation; Cancel confirms discarding dirty edits before returning to', 'the previous page; Export downloads the current saved preferences as JSON.', 'An Export failure uses the existing inline error/retry area and preserves', 'unsaved fields. Retry repeats Export; success clears only that Export error.', 'Retry controls are siblings beside the status text, outside its live region.', + 'Visible text stays “Retry”; its aria-label is “Retry save” or “Retry export” for that operation.', '', 'Responsive behavior: above 640px keep the header action group in one row; at', '640px and below, place full-width Save first and the three secondary actions',