From 5e7b66c7f41942ea2b0840405385612cec046d78 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Thu, 10 Sep 2026 13:01:01 +0000 Subject: [PATCH] test: recognize completed workflow evidence and bounded human waits --- test/autoplan-final-gate-ao.test.ts | 176 ++++++++++ test/autoplan-phase-dash-ao.test.ts | 78 +++++ test/ceo-test-subject-ao.test.ts | 83 +++++ test/conductor-prose-observation-ao.test.ts | 64 ++++ test/design-primary-assignment-ao.test.ts | 107 ++++++ test/design-primary-treatment-ao.test.ts | 125 +++++++ test/design-scope-announcement-ao.test.ts | 80 +++++ test/dx-manual-handoff-ao.test.ts | 107 ++++++ test/fixtures/autoplan-final-gate-ao.json | 322 ++++++++++++++++++ test/fixtures/autoplan-phase-dash-ao.json | 190 +++++++++++ test/fixtures/ceo-test-subject-ao.json | 259 ++++++++++++++ test/fixtures/conductor-prose-ao.json | 24 ++ .../design-primary-assignment-ao.json | 111 ++++++ .../fixtures/design-primary-treatment-ao.json | 113 ++++++ .../design-scope-announcement-ao.json | 61 ++++ test/fixtures/dx-manual-handoff-ao.json | 168 +++++++++ test/helpers/autoplan-phase-observer.ts | 10 +- test/helpers/autoplan-setup-question.ts | 52 ++- test/helpers/claude-pty-runner.ts | 75 +++- test/helpers/design-count-review.ts | 27 +- test/helpers/plan-scope-selection.ts | 16 +- test/helpers/touchfiles-data.ts | 24 +- test/skill-e2e-autoplan-chain.test.ts | 28 +- test/skill-e2e-conductor-prose.test.ts | 3 + 24 files changed, 2274 insertions(+), 29 deletions(-) create mode 100644 test/autoplan-final-gate-ao.test.ts create mode 100644 test/autoplan-phase-dash-ao.test.ts create mode 100644 test/ceo-test-subject-ao.test.ts create mode 100644 test/conductor-prose-observation-ao.test.ts create mode 100644 test/design-primary-assignment-ao.test.ts create mode 100644 test/design-primary-treatment-ao.test.ts create mode 100644 test/design-scope-announcement-ao.test.ts create mode 100644 test/dx-manual-handoff-ao.test.ts create mode 100644 test/fixtures/autoplan-final-gate-ao.json create mode 100644 test/fixtures/autoplan-phase-dash-ao.json create mode 100644 test/fixtures/ceo-test-subject-ao.json create mode 100644 test/fixtures/conductor-prose-ao.json create mode 100644 test/fixtures/design-primary-assignment-ao.json create mode 100644 test/fixtures/design-primary-treatment-ao.json create mode 100644 test/fixtures/design-scope-announcement-ao.json create mode 100644 test/fixtures/dx-manual-handoff-ao.json diff --git a/test/autoplan-final-gate-ao.test.ts b/test/autoplan-final-gate-ao.test.ts new file mode 100644 index 000000000..138018f0b --- /dev/null +++ b/test/autoplan-final-gate-ao.test.ts @@ -0,0 +1,176 @@ +import {expect, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import {autoplanBlockingQuestionBoundary, autoplanSetupDecision} from './helpers/autoplan-setup-question'; +import {autoplanPhaseCompletions} from './helpers/autoplan-phase-observer'; +import {readPendingQuestion, createPendingQuestionRecorder, recordPendingQuestion} from './helpers/plan-count-pending-question'; +import {readPlanCountTranscript} from './helpers/plan-count-transcript'; +import {E2E_TOUCHFILES} from './helpers/touchfiles'; +import capture from './fixtures/autoplan-final-gate-ao.json'; + +const fixture = (): {screen:string;context:Parameters[1]} => ({screen:capture.screen, context:{commandStartedAt:capture.commandStartedAt,viewportCapturedAt:capture.observedAt, + transcript:structuredClone(capture.transcript),publicTools:[structuredClone(capture.gateUse)]}}); +const detect = (f=fixture()) => autoplanBlockingQuestionBoundary(f.screen,f.context); +const gateCall = (f:ReturnType) => f.context.transcript.calls.find(c => c.toolUseId===capture.call.toolUseId)!; +function rebind(f:ReturnType) { f.context.publicTools[0]!.input!.questions=structuredClone(gateCall(f).questions); } + +test('exact AO unanswered gate stops observation but supplies no missing phase or approval', () => { + const f=fixture();const before=JSON.stringify(f); + expect(detect(f)).toEqual({sessionId:capture.call.sessionId,toolUseId:capture.call.toolUseId,source:'native'}); + expect(autoplanSetupDecision(f.screen,new Set(),gateCall(f)).kind).toBe('unrelated'); + // The separate dash repair recognizes DX; recorded original hits stay historical. + expect(autoplanPhaseCompletions(f.context.transcript,capture.commandStartedAt)).toEqual([ + ...capture.hits,{phase:2.5,ts:1789042284933}, + ]); + expect(capture.hits.map(h=>h.phase)).toEqual([1,2]); + expect(JSON.stringify(f)).toBe(before); + expect(gateCall(f).answered).toBe(false); +}); + +test('native question identity, status, chronology and current project are mandatory', () => { + const controls: Array<(f:ReturnType)=>void> = [ + f=>{f.context.transcript.status='missing';}, f=>{f.context.transcript.status='error';}, + f=>{f.context.publicTools=[];}, f=>{f.context.publicTools[0]!.timestamp='invalid';}, + f=>{f.context.commandStartedAt=Date.parse(capture.gateUse.timestamp)+1;}, + f=>{f.context.viewportCapturedAt=Date.parse(capture.gateUse.timestamp)-1;}, + f=>{f.context.publicTools[0]!.sessionId='foreign';}, f=>{f.context.publicTools[0]!.toolUseId='foreign';}, + f=>{f.context.publicTools[0]!.name='Read';}, + f=>{f.context.publicTools[0]!.input!.questions=[null];}, + f=>{f.context.publicTools[0]!.input!.questions=[{header:'Approval',question:'Partial'}];}, f=>{f.context.publicTools[0]!.input!.questions=[];}, + f=>{f.context.publicTools.push(structuredClone(f.context.publicTools[0]!));}, + f=>{f.context.publicTools.push({...f.context.publicTools[0]!,kind:'result',isError:false} as any);}, + f=>{gateCall(f).answered=true;}, f=>{gateCall(f).failed=true;}, + f=>{gateCall(f).sessionId='foreign';}, f=>{gateCall(f).questions[0]!.multiSelect=true;}, + f=>{gateCall(f).questions.push(structuredClone(gateCall(f).questions[0]!));}, + f=>{f.context.transcript.calls.push({...structuredClone(gateCall(f)),toolUseId:'other'});}, + f=>{f.context.commandStartedAt=NaN;}, + ]; + for(const [i,change] of controls.entries()){const f=fixture();change(f);expect(detect(f),String(i)).toBeNull();} +}); + +function render(f:ReturnType) { + const q=gateCall(f).questions[0]!;rebind(f); + f.screen=`☐ ${q.header}\n\n${q.question.split('\n').map(s=>'│ '+s).join('\n')}\n\n`+ + q.options.map((o,i)=>`${i===0?'❯ ': ' '}${i+1}. ${o.label}\n${o.description?.split('\n').map(s=>' '+s).join('\n')??''}`).join('\n')+ + `\n ${q.options.length+1}. Type something.\n ${q.options.length+2}. Chat about this\nEnter to select · ↑/↓ to navigate · Esc to cancel`; +} + +test('copied, stale and incomplete displays do not prove a current blocking question', () => { + for(const change of [ + (f:ReturnType)=>{f.screen='Source panel:\n'+f.screen;}, + f=>{f.screen='Example:\n'+f.screen;}, f=>{f.screen='```text\n'+f.screen+'\n```';}, + f=>{f.screen=f.screen.split('\n').map(row=>'> '+row).join('\n');}, + f=>{f.screen=f.screen.split('\n').map(row=>' '+row).join('\n');}, + f=>{f.screen+='\nContinuing the review.';}, f=>{f.screen=f.screen.replace('Esc to cancel','Esc to');}, + f=>{f.screen=f.screen.replace(' 6. Chat about this','');}, + f=>{f.screen=f.screen.replace('4. Revise the plan or reject','4. Unmatched current choice');}, + f=>{f.screen=f.screen.replace('D2 — Final Approval','D3 — Final Approval');}, + f=>{f.screen=f.screen.replace('❯ 1.',' 1.');}, + ]){const f=fixture();change(f);expect(detect(f)).toBeNull();} +}); + +test('an actual current human wait remains blocking regardless of source or withdrawn body semantics', () => { + for(const change of [ + (q:any)=>{q.question=q.question.replace('ELI10: ','ELI10: Source excerpt, not a current assessment: ');}, + (q:any)=>{q.question=q.question.replace('ELI10: ','ELI10: If approved, ');}, + (q:any)=>{q.question=q.question.replace('\nELI10:','\nSource excerpt:\nELI10:');}, + (q:any)=>{q.question+='\nThis final approval gate is cancelled.';}, + (q:any)=>{q.question+=' This approval gate is withdrawn.';}, + (q:any)=>{q.question+='\nCorrection: this final gate is not current.';}, + (q:any)=>{q.question+='\n> Historical note: the old gate was cancelled.';}, + (q:any)=>{q.question='Choose one of these approaches?';q.header='Approach';}, + (q:any)=>{q.question=q.question.replace(/^D2 /,'D9 ');}, + (q:any)=>{q.options[0].label='Start implementation';}, + ]){const f=fixture();change(gateCall(f).questions[0]);render(f);expect(detect(f)?.source).toBe('native');} +}); + +test('validated owned pending-hook fallback retains stale/foreign/completed rejection', () => { + const root=fs.mkdtempSync(path.join(os.tmpdir(),'autoplan-final-gate-')); + const cwd=path.join(root,path.basename(capture.cwd)),config=path.join(root,'config'); + fs.mkdirSync(cwd,{recursive:true});fs.mkdirSync(path.join(config,'projects','owned'),{recursive:true}); + const transcriptPath=path.join(config,'projects','owned',capture.call.sessionId+'.jsonl');fs.writeFileSync(transcriptPath,''); + const recorder=createPendingQuestionRecorder(cwd,config),startedAt=Date.now()-10; + const transcript:any={status:'ready',calls:[],assistantMessages:[{sessionId:capture.call.sessionId,timestamp:new Date(startedAt).toISOString(),text:'Finishing this review.'}]}; + const event={hook_event_name:'PreToolUse',cwd,session_id:capture.call.sessionId,tool_name:'AskUserQuestion',tool_use_id:capture.call.toolUseId,transcript_path:transcriptPath,tool_input:{questions:capture.call.questions}}; + try{ + recordPendingQuestion(JSON.stringify(event),recorder.file,cwd,config); + const get=(t=transcript,cwdArg=cwd,start=startedAt)=>readPendingQuestion(recorder.file,cwdArg,config,start,t); + const check=(pending=get(),t=transcript)=>autoplanBlockingQuestionBoundary(capture.screen,{commandStartedAt:startedAt,viewportCapturedAt:Date.now(),transcript:t,publicTools:[],pending}); + expect(check()?.source).toBe('pre_tool_use'); + expect(get(transcript,cwd+'-foreign')).toBeUndefined(); + expect(get(transcript,cwd,Date.now()+1000)).toBeUndefined(); + expect(get({...transcript,assistantMessages:[{...transcript.assistantMessages[0],sessionId:'foreign'}]})).toBeUndefined(); + for(const failed of [false,true]){ + const completed={...transcript,calls:[{...capture.call,answered:!failed,failed}]}; + expect(get(completed)).toBeUndefined();expect(check(undefined,completed)).toBeNull(); + } + recordPendingQuestion(JSON.stringify({...event,hook_event_name:'PostToolUse'}),recorder.file,cwd,config); + expect(get()).toBeUndefined();expect(check()).toBeNull(); + // The native route consumes the same cwd-scoped public reader as production. + // Only this local test envelope is synthetic; question bytes stay exact. + const record={cwd,sessionId:capture.call.sessionId,isSidechain:false,timestamp:new Date().toISOString(), + message:{role:'assistant',content:[{type:'tool_use',id:capture.call.toolUseId,name:'AskUserQuestion',input:{questions:capture.call.questions}}]}}; + const native=(owner=cwd)=>{ + const events:any[]=[];const transcript=readPlanCountTranscript(config,owner,e=>events.push(e)); + return autoplanBlockingQuestionBoundary(capture.screen,{commandStartedAt:startedAt,viewportCapturedAt:Date.now(),transcript,publicTools:events}); + }; + fs.writeFileSync(transcriptPath,JSON.stringify(record)+'\n'); + expect(native()?.source).toBe('native');expect(native(cwd+'-foreign')).toBeNull(); + fs.writeFileSync(transcriptPath,JSON.stringify({...record,isSidechain:true})+'\n');expect(native()).toBeNull(); + }finally{recorder.dispose();fs.rmSync(root,{recursive:true,force:true});} +}); + +test('production loop fails without answering; allowed and repeated setup keep their old behavior', async () => { + const source=fs.readFileSync(path.join(import.meta.dir,'skill-e2e-autoplan-chain.test.ts'),'utf8'); + const begin=source.indexOf(' // This new repository offers routing'); + const end=source.indexOf('\n }\n } finally',begin); + const block=source.slice(begin,end);expect(begin).toBeGreaterThan(0);expect(end).toBeGreaterThan(begin); + const AsyncFunction=Object.getPrototypeOf(async()=>{}).constructor; + const loop=new AsyncFunction('autoplanBlockingQuestionBoundary','autoplanSetupDecision','ctx', + new Bun.Transpiler({loader:'ts'}).transformSync(`async function observeBoundedLoop(){ + const {methodologyAudit,hits,commandStartedAt,viewportCapturedAt,transcript,publicTools,pendingSetupQuestion,panes}=ctx; + let outcome='timeout',evidence='',blockedQuestion=null,unsupportedSetup=null; + const inputs=[],seenSetupQuestions=new Set(),session={send:(s)=>inputs.push(s)},Bun={sleep:async()=>{}}; + const selectPtyNumberedOption=async(_session,n)=>session.send(String(n)+'\\r'),isPlanReadyVisible=()=>false; + for(const visible of panes){const viewport=visible;${block}} + return {outcome,blockedQuestion,hits,inputs};}`)+'return observeBoundedLoop();'); + const f=fixture(),ctx={...f.context,panes:[f.screen,f.screen],hits:structuredClone(capture.hits),methodologyAudit:['ceo','design','dx','eng'].map(phase=>({phase,passed:true}))}; + const run=(x=ctx)=>loop(autoplanBlockingQuestionBoundary,autoplanSetupDecision,x); + const result=await run();expect(result).toMatchObject({outcome:'blocked_on_question',hits:capture.hits,inputs:[]}); + expect(await run({...ctx,methodologyAudit:[{phase:'eng',passed:false}]})).toMatchObject({outcome:'incomplete_methodology',inputs:[]}); + expect(await run({...ctx,publicTools:[]})).toMatchObject({outcome:'timeout',inputs:[]}); + const partial=f.screen.replace('Esc to cancel','Esc to'); + expect(await run({...ctx,panes:[partial,partial]})).toMatchObject({outcome:'timeout',inputs:[]}); + expect(await run({...ctx,panes:[partial,f.screen]})).toMatchObject({outcome:'blocked_on_question',inputs:[]}); + const setup=fixture(),q=gateCall(setup).questions[0]!; + q.header='Routing';q.question='Add gstack skill routing rules to CLAUDE.md? '; + q.options=[{label:'Add routing rules (Recommended)',description:'Add project routing.'},{label:'Skip, invoke manually',description:'Keep manual invocation.'}];render(setup); + expect(autoplanSetupDecision(setup.screen,new Set(),gateCall(setup))).toMatchObject({kind:'input',input:'1'}); + const repeated=await run({...ctx,...setup.context,panes:[setup.screen,setup.screen]}); + expect(repeated).toMatchObject({outcome:'timeout',blockedQuestion:null,inputs:['1']}); + const complete=[1,2,2.5,3].map((phase,index)=>({phase,ts:capture.commandStartedAt+index+1})); + expect(await run({...ctx,hits:complete})).toMatchObject({outcome:'chain_complete',inputs:[]}); + const errorStart=source.indexOf(" if (outcome === 'blocked_on_question')"); + const errorEnd=source.indexOf(" if (outcome === 'exited'",errorStart); + const throwBlocked=new Function('outcome','hits','blockedQuestion','transcript','artifacts','evidence', + new Bun.Transpiler({loader:'ts'}).transformSync(source.slice(errorStart,errorEnd))); + expect(()=>throwBlocked(result.outcome,result.hits,result.blockedQuestion,f.context.transcript,{},'actual panel')).toThrow('missing phase markers=[2.5,3]'); + expect(()=>throwBlocked('blocked_on_question',[...ctx.hits,{phase:2.5,ts:capture.observedAt-1}],result.blockedQuestion,f.context.transcript,{},'actual panel')).toThrow('missing phase markers=[3]'); + // Even an impossible caller state with all markers cannot turn this disposition into success. + expect(()=>throwBlocked('blocked_on_question',complete,result.blockedQuestion,f.context.transcript,{},'actual panel')).toThrow('outcome=blocked_on_question'); + const validation=source.slice(source.indexOf(' // Phase 3 (Eng) MUST have been seen.'),source.indexOf(' } finally {\n try { fs.rmSync(tempDir',source.indexOf(' // Phase 3 (Eng) MUST have been seen.'))); + const validate=new Function('hits','methodologyAudit','expect','transcript','artifacts','evidence',new Bun.Transpiler({loader:'ts'}).transformSync(validation)); + const check=(hits:any[],audit=ctx.methodologyAudit)=>validate(hits,audit,expect,f.context.transcript,{},'Retained final gate'); + expect(()=>check(ctx.hits)).toThrow('Required phase markers missing');expect(()=>check(complete)).not.toThrow(); + expect(()=>check(complete,[])).toThrow(); + expect(()=>check(complete.map(h=>h.phase===2.5?{...h,ts:capture.commandStartedAt+10}:h))).toThrow(); +}); + +test('only the Autoplan owner adds the exact fixtures and every indexed entry stays dense', () => { + expect(E2E_TOUCHFILES['autoplan-chain-pty']).toContain('test/autoplan-final-gate-ao.test.ts'); + expect(E2E_TOUCHFILES['autoplan-chain-pty']).toContain('test/fixtures/autoplan-final-gate-ao.json'); + for(const paths of Object.values(E2E_TOUCHFILES))for(let i=0;i ({ + status: 'ready', calls: [], assistantMessages: [{ ...fixture.message, text }], +}); +const hits = (text: string) => autoplanPhaseCompletions(transcript(text), at - 1); + +test('exact owned DX dash declaration adds only DX at its native timestamp', () => { + expect(hits(fixture.message.text)).toEqual([{ phase: 2.5, ts: at }]); + const all = autoplanPhaseCompletions({ status: 'ready', calls: [], + assistantMessages: fixture.orderedMessages }, fixture.commandLowerBound); + expect(all).toEqual([...fixture.actualHits, { phase: 2.5, ts: at }]); + expect(all.map(hit => hit.phase)).toEqual([1, 2, 2.5]); +}); + +test('em and en dash spacing share the existing completed declaration forms', () => { + for (const dash of ['—', '–']) for (const before of ['', ' ']) for (const after of ['', ' ']) { + expect(hits(fixture.message.text.replace('complete—', `complete${before}${dash}${after}`))) + .toEqual([{ phase: 2.5, ts: at }]); + for (const phase of [1, 2, 2.5, 3]) for (const state of ['complete', 'completed', 'done', 'finished', 'wrapped up']) { + expect(hits(`Phase ${phase} is ${state}${before}${dash}${after}Work retained.`)) + .toEqual([{ phase, ts: at }]); + } + } + expect(hits('**Phase 2.5 complete** — Work retained.')).toEqual([{ phase: 2.5, ts: at }]); +}); + +test('dash continuations cannot turn a conditional, quotation, question or denial into completion', () => { + for (const dash of ['—', '–']) for (const tail of [ + '', 'if approved.', 'unless the checks fail.', 'when review finishes.', + 'once the reviewer signs off.', 'pending final checks.', 'maybe tomorrow.', + 'perhaps it is complete.', 'would be complete after review.', + 'not complete yet.', 'the phase is not complete.', 'this completion is withdrawn.', + 'actually never finished.', 'this completion is superseded.', + 'provided the remaining checks pass.', 'this completion is rejected.', + 'the completion announcement is retracted.', 'actually incomplete.', + 'the review remains pending.', 'Work retained?', 'is this complete?', + 'Source excerpt: Work retained.', 'Earlier review: Work retained.', + 'the historical example says work is retained.', '"Work retained."', + ]) expect(hits(`Phase 2.5 complete ${dash} ${tail}`), tail).toEqual([]); + for (const text of [ + 'If approved, Phase 2.5 complete—Work retained.', + 'Phase 2.5 is not complete—Work retained.', + 'Phase 2.5 complete?—Work retained.', + '> Phase 2.5 complete—Work retained.', + '"Phase 2.5 complete—Work retained."', + 'Source excerpt:\nPhase 2.5 complete—Work retained.', + 'Example:\nPhase 2.5 complete—Work retained.\nPhase 3 complete—Work retained.', + '```text\nPhase 2.5 complete—Work retained.\n```', + ' Phase 2.5 complete—Work retained.', + '# Phase 2.5 complete—Work retained.', + 'Phase 2.5 (Eng review) complete—Work retained.', + ]) expect(hits(text), text).toEqual([]); +}); + +test('dash support keeps ready/current native evidence and first-hit ordering', () => { + for (const status of ['missing', 'error'] as const) { + expect(autoplanPhaseCompletions({ ...transcript(), status }, at - 1)).toEqual([]); + } + expect(autoplanPhaseCompletions(transcript(), at + 1)).toEqual([]); + expect(autoplanPhaseCompletions({ ...transcript(), assistantMessages: [ + { ...fixture.message, timestamp: 'invalid' }, + ] }, at - 1)).toEqual([]); + const later = { ...fixture.message, timestamp: new Date(at + 1).toISOString() }; + expect(autoplanPhaseCompletions({ ...transcript(), assistantMessages: [later, fixture.message] }, at - 1)) + .toEqual([{ phase: 2.5, ts: at }]); +}); + +test('dash fixture and regression select only the existing AP owner', () => { + for (const file of ['test/autoplan-phase-dash-ao.test.ts', 'test/fixtures/autoplan-phase-dash-ao.json']) { + expect(selectTests([file], E2E_TOUCHFILES).selected).toEqual(['autoplan-chain-pty']); + } +}); diff --git a/test/ceo-test-subject-ao.test.ts b/test/ceo-test-subject-ao.test.ts new file mode 100644 index 000000000..067ed5e15 --- /dev/null +++ b/test/ceo-test-subject-ao.test.ts @@ -0,0 +1,83 @@ +import { expect, test } from 'bun:test'; +import { ceoFirstReviewAUQ, ceoStep0Boundary, planCountQuestionPhase, type AskUserQuestionFingerprint } from './helpers/claude-pty-runner'; +import { E2E_TOUCHFILES } from './helpers/touchfiles-data'; +import fixture from './fixtures/ceo-test-subject-ao.json'; +const calls=fixture.fingerprints as AskUserQuestionFingerprint[]; +const actual=calls[1]!; +function change(edit:(q:any, call:any, fp:any)=>void) { + const fp=structuredClone(actual),call=fp.nativeCall!,q=call.questions[0]!; + const selected=q.options.findIndex(o=>o.label===call.answers?.[q.question]); + edit(q,call,fp); + call.answers={[q.question]:q.options[selected]?.label??''}; + fp.options=q.options.map((o,i)=>({index:i+1,label:o.label})); + return fp; +} +test('the completed affected-Test question starts review from its current ELI10 assertion gap',()=>{ + expect(calls.map(ceoFirstReviewAUQ)).toEqual([false,true,false]); + let review=false; + expect(calls.map(fp=>{const phase=planCountQuestionPhase(fp,review,ceoStep0Boundary,ceoFirstReviewAUQ);review=phase.reviewStarted;return phase.preReview;})).toEqual([true,false,false]); +}); +test('structural Test identity permits ordinary question and separator variations',()=>{ + for(const title of [ + 'D4 — Test 1: choose its assertion?', + 'D4 — Test 1 — which assertion belongs here?', + 'D4 - Test 1 (successful charge): what must this test verify?', + 'd4 — Test 1 (successful charge): assertion choice?', + 'D4 — Test 1 what should the expected result be?', + ]) expect(ceoFirstReviewAUQ(change(q=>{q.question=q.question.replace(/^[^\n]+/,title);}))).toBe(true); + expect(ceoFirstReviewAUQ(change(q=>{ + q.question=q.question.replace(/^D4/,'D17').replace(/\b4([A-C])\b/g,'17$1'); + q.options=q.options.map((o:any)=>({...o,label:o.label.replace(/^4/,'17')})); + }))).toBe(true); +}); +test('test headers, competing finding IDs and uniform foreign decision choices cannot borrow the assessment',()=>{ + for(const header of ['Test 2','Finding 1','Issue 1','Approach']) expect(ceoFirstReviewAUQ(change(q=>{q.header=header;}))).toBe(false); + expect(ceoFirstReviewAUQ(change(q=>{q.question=q.question.replace('Test 1 (successful charge)','Test 1 (Finding 2)');}))).toBe(false); + expect(ceoFirstReviewAUQ(change(q=>{ + q.question=q.question.replace(/\b4([A-C])\b/g,'8$1');q.options=q.options.map((o:any)=>({...o,label:o.label.replace(/^4/,'8')})); + }))).toBe(false); +}); +test('explicit Test and decision identifiers must be anchored integers with one test owner',()=>{ + for(const header of ['Test 0','Test 01','Test 1.2']) expect(ceoFirstReviewAUQ(change(q=>{q.header=header;}))).toBe(false); + for(const decision of ['D0','D04']) expect(ceoFirstReviewAUQ(change(q=>{q.question=q.question.replace(/^D4/,decision);}))).toBe(false); + expect(ceoFirstReviewAUQ(change(q=>{q.question=q.question.replace('Test 1 (successful charge)','Test 1 (Test 2)');}))).toBe(false); + for(const header of ['Receipt assertion','Test contract','Test 1: receipt assertion']) expect(ceoFirstReviewAUQ(change(q=>{q.header=header;}))).toBe(true); + expect(ceoFirstReviewAUQ(change(q=>{q.question=q.question.replace('Test 1 (successful charge)','Test 1 ("Test 2" is an archive label)');}))).toBe(true); +}); +test('the owned weak assertion must remain current and outside quoted or conditional source frames',()=>{ + for(const intro of ['Source excerpt: ','Earlier review assessment: ','If approved later, ']) expect(ceoFirstReviewAUQ(change(q=>{ + q.question=q.question.replace('The planned test only checks',intro+'The planned test only checks'); + }))).toBe(false); + for(const intro of ['Source excerpt follows. ','Earlier review assessment follows. ','If approved later. ']) expect(ceoFirstReviewAUQ(change(q=>{ + q.question=q.question.replace('The planned test only checks',intro+'The planned test only checks'); + }))).toBe(false); + expect(ceoFirstReviewAUQ(change(q=>{q.question=q.question.replace('The planned test only checks','The planned test no longer only checks');}))).toBe(false); + expect(ceoFirstReviewAUQ(change(q=>{q.question=q.question.replace('The planned test only checks that the receipt is truthy.','"The planned test only checks that the receipt is truthy."');}))).toBe(false); +}); +test('direct current withdrawals stay effective while a quoted historical note stays harmless',()=>{ + for(const text of ['This finding is withdrawn.','This assessment is "closed".','Correction: this explanation is not current.']) expect(ceoFirstReviewAUQ(change(q=>{q.question+='\n'+text;}))).toBe(false); + expect(ceoFirstReviewAUQ(change(q=>{q.question=q.question.replace('\nELI10:','\nArchive note: "Source: this finding is withdrawn."\nELI10:');}))).toBe(true); +}); +test('the complete current amendment belongs to an offered option',()=>{ + for(const prefix of ['Source excerpt: ','Historical example: ','If approved later: ']) expect(ceoFirstReviewAUQ(change(q=>{ + for(const o of q.options)o.description=prefix+o.description; + }))).toBe(false); + for(const text of [' This amendment is withdrawn.',' This amendment is "closed".',' This remedy is a historical example, not the current option.']) expect(ceoFirstReviewAUQ(change(q=>{ + for(const o of q.options)o.description+=text; + }))).toBe(false); + expect(ceoFirstReviewAUQ(change(q=>{q.options[0].label='4A: Keep the truthy assertion (recommended)';}))).toBe(false); +}); +test('native completion, exact answer, index and menu identity remain required',()=>{ + for(const edit of [ + (_q:any,c:any)=>{c.answered=false;},(_q:any,c:any)=>{c.failed=true;}, + (_q:any,c:any)=>{delete c.answeredAt;},(_q:any,c:any)=>{c.unansweredQuestionIndices=[0];}, + (_q:any,_c:any,fp:any)=>{fp.signature='foreign:tool';}, + (_q:any,_c:any,fp:any)=>{fp.nativeQuestionIndex=1;},(q:any)=>{q.multiSelect=true;}, + ]) expect(ceoFirstReviewAUQ(change(edit))).toBe(false); + const wrongAnswer=change(()=>{});wrongAnswer.nativeCall!.answers={};expect(ceoFirstReviewAUQ(wrongAnswer)).toBe(false); + const wrongMenu=change(()=>{});wrongMenu.options[0]!.label='Foreign menu';expect(ceoFirstReviewAUQ(wrongMenu)).toBe(false); +}); +test('new inputs belong only to the dense CEO finding owner',()=>{ + for(const file of ['test/ceo-test-subject-ao.test.ts','test/fixtures/ceo-test-subject-ao.json']) expect(Object.entries(E2E_TOUCHFILES).filter(([,paths])=>paths.includes(file)).map(([owner])=>owner)).toEqual(['plan-ceo-finding-count']); + for(const paths of Object.values(E2E_TOUCHFILES))for(let i=0;iframes[Math.min(Math.max(tick,0),frames.length-1)]!; + const args:Record={path,process:{cwd:()=>'/synthetic-owned'},Date:{now:()=>clock},randomUUID:()=> 'owned', + Bun:{sleep:async(ms:number)=>{if(ms===2000){tick++;clock+=61000;}else clock+=ms;}}, + launchClaudePty:async()=>({send:()=>{},mark:()=>0,exited:()=>false,visibleSince:current,rawOutput:current,currentScreen:async()=>current(),hermeticConfigDir:null,close:async()=>{closed++;}}), + createPlanCountSnapshotWriter:()=>()=>({}),logPtySnapshot:()=>{}, + isProseAUQVisible:predicates.isProseAUQVisible,isPlanReadyVisible:predicates.isPlanReadyVisible, + isScopeGateQuestionVisible:predicates.isScopeGateQuestionVisible,isScopeGateAutoSelectVisible:predicates.isScopeGateAutoSelectVisible, + classifyVisible:predicates.classifyVisible,extractPlanFilePath:predicates.extractPlanFilePath,findNativeAutoDecision:()=>null, + judgePtyState:()=>{judged++;return {state:verdict,reasoning:'synthetic fixed verdict'};}, + }; + const run=new Function(...Object.keys(args),js)(...Object.values(args)); + const obs=await run({skillName:'plan-eng-review',initialPlanContent:'# Plan: Required draft',timeoutMs:300000,...(required===undefined?{}:{requireProseEvidence:required})}); + expect(closed).toBe(1); + return {obs,polls:tick+1,judged}; +} + +test('a judge waiting on the exact partial Conductor brief cannot stop a prose-required observation',async()=>{ + expect(fixture.actualFlags.proseAUQEverObserved).toBe(false);expect(fixture.actualFlags.waitingEverObserved).toBe(true); + expect(predicates.isProseAUQVisible(partial)).toBe(false);expect(predicates.isProseAUQVisible(complete)).toBe(true); + const {obs,polls,judged}=await observe([partial,complete],'waiting',true); + expect(polls).toBe(2);expect(judged).toBe(1);expect(obs.outcome).toBe('asked'); + expect(obs.proseAUQEverObserved).toBe(true);expect(obs.waitingEverObserved).toBe(true); +}); +test('partial-only judge waiting reaches the existing budget without gaining prose fallback credit',async()=>{ + const {obs,polls,judged}=await observe([partial],'waiting',true); + expect(polls).toBe(5);expect(judged).toBe(5);expect(obs.outcome).toBe('timeout'); + expect(obs.proseAUQEverObserved).toBe(false);expect(obs.waitingEverObserved).toBe(true); +}); +test('the prose requirement does not alter completed questions or deterministic failure precedence',async()=>{ + const done=await observe([complete],'waiting',true); + expect(done.obs.outcome).toBe('asked');expect(done.obs.proseAUQEverObserved).toBe(true);expect(done.judged).toBe(0); + const wrote=await observe(['⏺ Write(/tmp/foreign-output.md)'],'waiting',true); + expect(wrote.obs.outcome).toBe('silent_write');expect(wrote.obs.proseAUQEverObserved).toBe(false);expect(wrote.judged).toBe(0); +}); +test('other callers retain the original judge waiting behavior',async()=>{ + for(const required of [undefined,false]){ + const {obs,polls}=await observe([partial,complete],'waiting',required); + expect(polls).toBe(1);expect(obs.outcome).toBe('asked');expect(obs.proseAUQEverObserved).toBe(false);expect(obs.waitingEverObserved).toBe(true); + } + const working=await observe([partial],'working',true); + expect(working.obs.outcome).toBe('timeout');expect(working.obs.waitingEverObserved).toBe(false); +}); +test('the actual Conductor caller requests prose evidence and retains its independent assertion',()=>{ + const caller=fs.readFileSync(path.join(import.meta.dir,'skill-e2e-conductor-prose.test.ts'),'utf8'); + expect(caller).toContain('requireProseEvidence: true'); + expect(caller).toContain('expect(obs.proseAUQEverObserved).toBe(true)'); + for(const p of ['test/conductor-prose-observation-ao.test.ts','test/fixtures/conductor-prose-ao.json'])expect(Object.entries(E2E_TOUCHFILES).filter(([,paths])=>paths.includes(p)).map(([owner])=>owner)).toEqual(['conductor-prose']); +}); diff --git a/test/design-primary-assignment-ao.test.ts b/test/design-primary-assignment-ao.test.ts new file mode 100644 index 000000000..ffe31883d --- /dev/null +++ b/test/design-primary-assignment-ao.test.ts @@ -0,0 +1,107 @@ +import { expect, test } from 'bun:test'; +import captured from './fixtures/design-primary-assignment-ao.json'; +import { isDesignCountFirstReview, isDesignCountSetup } from './helpers/design-count-review'; +import { designStep0Boundary, planCountQuestionPhase } from './helpers/claude-pty-runner'; +import type { AskUserQuestionFingerprint as Fingerprint } from './helpers/claude-pty-runner'; +import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles'; + +type Question = NonNullable['questions'][number]; +function edit(change: (q: Question) => void): Fingerprint { + const fp = structuredClone(captured.fingerprints[0]) as Fingerprint; + const call = fp.nativeCall!, q = call.questions[0]!; + const selected = q.options.findIndex(o => o.label === call.answers![q.question]); + change(q); + call.answers = { [q.question]: q.options[selected]!.label }; + fp.options = q.options.map((o, i) => ({ index: i + 1, label: o.label })); + return fp; +} + +test('the exact style assignment begins review before the following pending-state decision', () => { + let started = false; + const phases = captured.fingerprints.map(raw => { + const phase = planCountQuestionPhase(raw as Fingerprint, started, designStep0Boundary, + isDesignCountFirstReview, isDesignCountSetup); + started = phase.reviewStarted; + return phase; + }); + expect(phases).toEqual([ + { preReview: false, reviewStarted: true }, + { preReview: false, reviewStarted: true }, + ]); + expect(captured.fingerprints.map(fp => fp.preReview)).toEqual([true, true]); +}); + +test('assignment whitespace and current deferral phrasing compose', () => { + for (const separator of [' = ', '=', ' =']) { + for (const action of ['Leave', 'Keep']) { + for (const debt of ['debt', 'an open issue']) { + expect(isDesignCountFirstReview(edit(q => { + q.options[0]!.description = q.options[0]!.description!.replaceAll(' = ', separator); + q.options[2]!.description = q.options[2]!.description!.replace('Leave the header', `${action} the header`) + .replace('as debt.', `as ${debt}.`); + }))).toBe(true); + } + } + } +}); + +test('the named control and offered answer may change without changing the review phase', () => { + expect(isDesignCountFirstReview(edit(q => { + q.question = q.question.replaceAll('Save', 'Submit'); + q.options = q.options.map(o => ({ label: o.label.replaceAll('Save', 'Submit'), + description: o.description?.replaceAll('Save', 'Submit') })); + }))).toBe(true); + for (const selected of [0, 1, 2]) { + const fp = edit(() => {}), q = fp.nativeCall!.questions[0]!; + fp.nativeCall!.answers = { [q.question]: q.options[selected]!.label }; + expect(isDesignCountFirstReview(fp)).toBe(true); + } +}); + +const rejected: Array<[string, (q: Question) => void]> = [ + ['withdrawn contract', q => { q.question += '\nThis DESIGN.md contract is "withdrawn".'; }], + ['superseded contract', q => { q.question += '\nThis contract is "superseded".'; }], + ['wrong primary', q => { q.options[0]!.description = q.options[0]!.description!.replace('Save =', 'Reset ='); }], + ['primary also ghost', q => { q.options[0]!.description = q.options[0]!.description!.replace('Reset/Cancel/Export =', 'Save/Cancel/Export ='); }], + ['no primary foreground', q => { q.options[0]!.description = q.options[0]!.description!.replace(' with white text', ''); }], + ['no ghost treatment', q => { q.options[0]!.description = q.options[0]!.description!.replace('neutral ghost Buttons', 'filled Buttons'); }], + ['no current authority', q => { q.options[0]!.description = q.options[0]!.description!.replace('per DESIGN.md', 'per a draft proposal'); }], + ['conditional assignment', q => { q.options[0]!.description = 'If approved later: ' + q.options[0]!.description; }], + ['historical assignment', q => { q.options[0]!.description = 'Historical example: ' + q.options[0]!.description; }], + ['quoted assignment', q => { q.options[0]!.description = '> ' + q.options[0]!.description; }], + ['cancelled assignment', q => { q.options[0]!.description += '\nCorrection: do not apply these styles.'; }], + ['rejected assignment', q => { q.options[0]!.description += '\nThis amendment is "rejected".'; }], + ['no opposed option', q => { q.options[2]!.label = '1C Configure Export'; }], + ['no documented violation', q => { q.options[2]!.description = q.options[2]!.description!.replace('Ships a known DESIGN.md violation', 'Satisfies DESIGN.md'); }], + ['no remaining primary gap', q => { q.options[2]!.description = q.options[2]!.description!.replace('stays undiscoverable', 'becomes obvious'); }], + ['conditional deferral', q => { q.options[2]!.description = 'If approved later: ' + q.options[2]!.description; }], + ['historical deferral', q => { q.options[2]!.description = 'Historical example: ' + q.options[2]!.description; }], + ['quoted deferral', q => { q.options[2]!.description = '> ' + q.options[2]!.description; }], + ['cancelled deferral', q => { q.options[2]!.description += '\nThis deferral is "cancelled".'; }], + ['cancelled header instruction', q => { q.options[2]!.description += '\nCorrection: do not leave the header unchanged.'; }], + ['cancelled keep instruction', q => { q.options[2]!.description += '\nCorrection: do not keep the header unchanged.'; }], + ['resolved violation', q => { q.options[2]!.description += '\nThis violation is now resolved.'; }], + ['conditional benefit', q => { q.options[2]!.description = q.options[2]!.description!.replace('✅ Zero implementation', '✅ If approved later: zero implementation'); }], +]; +test.each(rejected)('%s cannot start review', (_, change) => { + expect(isDesignCountFirstReview(edit(change))).toBe(false); +}); + +test('a quoted historical cancellation does not cancel the current deferral', () => { + expect(isDesignCountFirstReview(edit(q => { + q.options[2]!.description += '\nHistorical note: "Correction: do not leave the header unchanged."'; + }))).toBe(true); +}); + +test('unfinished or foreign native calls cannot start review', () => { + const incomplete = edit(() => {}); incomplete.nativeCall!.answered = false; + expect(isDesignCountFirstReview(incomplete)).toBe(false); + const foreign = edit(() => {}); foreign.signature = 'foreign:tool'; + expect(isDesignCountFirstReview(foreign)).toBe(false); +}); + +test('the retry regression maps only to the existing Design workflow owner', () => { + for (const file of ['test/design-primary-assignment-ao.test.ts', 'test/fixtures/design-primary-assignment-ao.json']) { + expect(selectTests([file], E2E_TOUCHFILES, []).selected).toEqual(['plan-design-finding-count']); + } +}); diff --git a/test/design-primary-treatment-ao.test.ts b/test/design-primary-treatment-ao.test.ts new file mode 100644 index 000000000..41e1f24a5 --- /dev/null +++ b/test/design-primary-treatment-ao.test.ts @@ -0,0 +1,125 @@ +import { expect, test } from 'bun:test'; +import captured from './fixtures/design-primary-treatment-ao.json'; +import { isDesignCountFirstReview, isDesignCountSetup } from './helpers/design-count-review'; +import { designStep0Boundary, planCountQuestionPhase } from './helpers/claude-pty-runner'; +import type { AskUserQuestionFingerprint as Fingerprint } from './helpers/claude-pty-runner'; +import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles'; + +type Question = NonNullable['questions'][number]; +const original = () => structuredClone(captured.fingerprints[0]) as Fingerprint; +function edit(change: (q: Question) => void): Fingerprint { + const fp = original(), call = fp.nativeCall!, q = call.questions[0]!; + const selected = q.options.findIndex(o => o.label === call.answers![q.question]); + change(q); + call.answers = { [q.question]: q.options[selected]!.label }; + fp.options = q.options.map((o, i) => ({ index: i + 1, label: o.label })); + return fp; +} + +test('the exact first primary-treatment decision starts review and retains the following decision', () => { + expect(isDesignCountFirstReview(original())).toBe(true); + let started = false; + const phases = captured.fingerprints.map(raw => { + const phase = planCountQuestionPhase(raw as Fingerprint, started, designStep0Boundary, + isDesignCountFirstReview, isDesignCountSetup); + started = phase.reviewStarted; + return phase; + }); + expect(phases).toEqual([ + { preReview: false, reviewStarted: true }, + { preReview: false, reviewStarted: true }, + ]); + expect(captured.fingerprints.map(fp => fp.preReview)).toEqual([true, true]); +}); + +test('equivalent primary qualifiers, singular filled treatment and authority compose', () => { + for (const qualifier of ['visible', 'visually', 'single filled']) { + for (const treatment of ['one filled button', 'single filled primary', 'one filled primary button']) { + for (const authority of ['per DESIGN.md', 'exactly as DESIGN.md specifies']) { + expect(isDesignCountFirstReview(edit(q => { + q.question = q.question.replace('visually primary', `${qualifier} primary`); + q.options[0]!.description = q.options[0]!.description! + .replace('one filled button', treatment).replace('per DESIGN.md', authority); + }))).toBe(true); + } + } + } +}); + +test('a different control or an opposed answer preserves the current decision', () => { + expect(isDesignCountFirstReview(edit(q => { + q.question = q.question.replaceAll('Save', 'Submit'); + q.options = q.options.map(o => ({ label: o.label.replaceAll('Save', 'Submit'), + description: o.description?.replaceAll('Save', 'Submit') })); + }))).toBe(true); + for (const option of original().nativeCall!.questions[0]!.options) { + const fp = original(), call = fp.nativeCall!; + call.answers = { [call.questions[0]!.question]: option.label }; + expect(isDesignCountFirstReview(fp)).toBe(true); + } +}); + +const rejected: Array<[string, (q: Question) => void]> = [ + ['foreign header', q => { q.header = 'Issue 2'; }], + ['workflow title', q => { q.question = q.question.replace('Make Save the visually primary action in the header', 'Run outside design voices'); }], + ['historical question', q => { q.question = 'Historical example:\n' + q.question; }], + ['source assessment', q => { q.question = q.question.replace('\nELI10:', '\nSource excerpt:\nELI10:'); }], + ['quoted assessment', q => { q.question = q.question.replace('\nELI10:', '\n> ELI10:'); }], + ['conditional assessment', q => { q.question = q.question.replace('ELI10: Right now', 'ELI10: If right now'); }], + ['no current equal-weight gap', q => { q.question = q.question.replace('all look identical', 'do not look identical'); }], + ['withdrawn contract', q => { q.question += '\nThis DESIGN.md contract is withdrawn.'; }], + ['quoted withdrawn contract', q => { q.question += '\nThis DESIGN.md contract is "withdrawn".'; }], + ['superseded requirement', q => { q.question += '\nThis requirement is superseded.'; }], + ['quoted superseded requirement', q => { q.question += '\nThis requirement is \"superseded\".'; }], + ['rejected contract', q => { q.question += '\nThis DESIGN.md contract is rejected.'; }], + ['quoted cancelled contract', q => { q.question += '\nThis DESIGN.md contract is \"cancelled\".'; }], + ['withdrawn issue', q => { q.question += '\nThis issue is withdrawn.'; }], + ['quoted rejected issue', q => { q.question += '\nThis issue is "rejected".'; }], + ['wrong named primary', q => { q.options[0]!.description = q.options[0]!.description!.replace('Save becomes', 'Reset becomes'); }], + ['primary also ghost', q => { q.options[0]!.description = q.options[0]!.description!.replace('; Reset,', '; Save,'); }], + ['missing foreground', q => { q.options[0]!.description = q.options[0]!.description!.replace(', white text', ''); }], + ['missing ghost treatment', q => { q.options[0]!.description = q.options[0]!.description!.replace('neutral ghost buttons', 'filled buttons'); }], + ['missing style authority', q => { q.options[0]!.description = q.options[0]!.description!.replace('per DESIGN.md', 'per a future proposal'); }], + ['conditional amendment', q => { q.options[0]!.description = 'If approved later: ' + q.options[0]!.description; }], + ['quoted amendment', q => { q.options[0]!.description = '> ' + q.options[0]!.description; }], + ['cancelled amendment', q => { q.options[0]!.description += '\nCorrection: do not apply these styles.'; }], + ['quoted rejected amendment', q => { q.options[0]!.description += '\nThis amendment is "rejected".'; }], + ['no opposed choice', q => { q.options[2]!.label = '1C Configure Export'; }], + ['opposed gap closed', q => { q.options[2]!.description = q.options[2]!.description!.replace('gap stays open', 'gap is closed'); }], + ['historical opposed choice', q => { q.options[2]!.description = 'Historical example: ' + q.options[2]!.description; }], + ['conditional opposed choice', q => { q.options[2]!.description = 'If approved later: ' + q.options[2]!.description; }], + ['quoted opposed choice', q => { q.options[2]!.description = '> ' + q.options[2]!.description; }], + ['resolved gap', q => { q.options[2]!.description += '\nThe gap is now resolved.'; }], + ['quoted rejected opposed choice', q => { q.options[2]!.description += '\nThis option is "rejected".'; }], +]; +test.each(rejected)('%s does not start review', (_, change) => { + expect(isDesignCountFirstReview(edit(change))).toBe(false); +}); + +test('a wholly quoted historical cancellation does not withdraw this requirement', () => { + expect(isDesignCountFirstReview(edit(q => { + q.question += '\nHistorical note: "This DESIGN.md contract is withdrawn."'; + }))).toBe(true); +}); + +test('recognition requires the same completed native identity and offered answer', () => { + for (const change of [ + (fp: Fingerprint) => { fp.nativeCall!.answered = false; }, + (fp: Fingerprint) => { fp.nativeCall!.failed = true; }, + (fp: Fingerprint) => { fp.signature = 'foreign:tool'; }, + (fp: Fingerprint) => { fp.nativeQuestionIndex = 1; }, + (fp: Fingerprint) => { fp.nativeCall!.unansweredQuestionIndices = [0]; }, + (fp: Fingerprint) => { delete fp.nativeCall!.answeredAt; }, + (fp: Fingerprint) => { fp.nativeCall!.answers = {}; }, + (fp: Fingerprint) => { fp.options.reverse(); }, + ]) { + const fp = original(); change(fp); + expect(isDesignCountFirstReview(fp)).toBe(false); + } +}); + +test('both source regressions select the existing Design workflow owner', () => { + for (const file of ['test/design-primary-treatment-ao.test.ts', 'test/fixtures/design-primary-treatment-ao.json']) { + expect(selectTests([file], E2E_TOUCHFILES, []).selected).toEqual(['plan-design-finding-count']); + } +}); diff --git a/test/design-scope-announcement-ao.test.ts b/test/design-scope-announcement-ao.test.ts new file mode 100644 index 000000000..2e412e9aa --- /dev/null +++ b/test/design-scope-announcement-ao.test.ts @@ -0,0 +1,80 @@ +import { expect, test } from 'bun:test'; +import captured from './fixtures/design-scope-announcement-ao.json'; +import { nativeSeededPlanSelection } from './helpers/plan-scope-selection'; +import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles'; +import type { PlanCountTranscript, NativePublicToolEvent } from './helpers/plan-count-transcript'; + +const input = () => structuredClone(captured.projection); +type Input = ReturnType; +const announcement = (p: Input) => p.transcript.assistantMessages.find(m => m.sessionId === p.opts.sessionId && m.text.startsWith("I'll auto-select"))!; +const verdict = (p: Input) => nativeSeededPlanSelection(p.transcript as PlanCountTranscript, p.tools as NativePublicToolEvent[], p.opts); + +test('the exact owned post-load option B announcement selects the seeded title', () => { + expect(captured.rawScopeGateAutoSelectObserved).toBe(false); + expect(verdict(input())).toBe(true); +}); + +test('equivalent explicit selection words and balanced title quotes retain identity', () => { + for (const prefix of ["I'll auto-select", 'I will auto-select', "I'll auto select"]) { + for (const title of ['Marketing landing page', '"Marketing landing page"', '“Marketing landing page”', '`Marketing landing page`']) { + const p = input(), m = announcement(p); + m.text = m.text.replace("I'll auto-select", prefix).replace('Marketing landing page', title); + expect(verdict(p)).toBe(true); + } + } + const p = input(), m = announcement(p); + p.opts.seed = p.opts.seed.replace('Marketing landing page', 'Account settings'); + m.text = m.text.replace('Marketing landing page', 'Account settings'); + expect(verdict(p)).toBe(true); +}); + +const rejected: Array<[string, (p: Input) => void]> = [ + ['wrong option', p => { announcement(p).text = announcement(p).text.replace('option B', 'option A'); }], + ['wrong target', p => { announcement(p).text = announcement(p).text.replace('Marketing landing page', 'Account settings'); }], + ['target prefix only', p => { announcement(p).text = announcement(p).text.replace('page draft', 'page experiment draft'); }], + ['conditional selection', p => { announcement(p).text = 'If approved: ' + announcement(p).text; }], + ['source selection', p => { announcement(p).text = 'Source excerpt:\n' + announcement(p).text; }], + ['quoted selection', p => { announcement(p).text = '> ' + announcement(p).text; }], + ['wholly quoted selection', p => { announcement(p).text = '"' + announcement(p).text + '"'; }], + ['unbalanced target quotes', p => { announcement(p).text = announcement(p).text.replace('Marketing landing page', '"Marketing landing page'); }], + ['question instead of assertion', p => { announcement(p).text = announcement(p).text.replace(/\.$/, '?'); }], + ['conditional tail', p => { announcement(p).text = announcement(p).text.replace(', running', ' if approved, running'); }], + ['cancelled selection', p => { announcement(p).text += '\nCorrection: this selection is withdrawn.'; }], + ['quoted status cancellation', p => { announcement(p).text += '\nThis selection is "withdrawn".'; }], + ['replaced target', p => { announcement(p).text += '\nThe selected target is now the branch diff.'; }], + ['pre-load announcement', p => { announcement(p).timestamp = new Date(Date.parse(p.tools[1]!.timestamp) - 1).toISOString(); }], + ['foreign announcement', p => { announcement(p).sessionId = 'foreign'; }], + ['foreign load result', p => { p.tools[1]!.sessionId = 'foreign'; }], + ['failed skill load', p => { p.tools[1]!.isError = true; }], + ['wrong skill', p => { p.tools[0]!.input!.skill = 'plan-eng-review'; }], + ['late command start', p => { p.opts.commandStartedAt = Date.parse(p.tools[1]!.timestamp) + 1; }], + ['multiple seed titles', p => { p.opts.seed += '\n# Another plan\n'; }], +]; +test.each(rejected)('%s supplies no scope selection', (_, change) => { + const p = input(); change(p); expect(verdict(p)).toBe(false); +}); + +test('quoted historical or foreign withdrawals do not replace the current selection', () => { + for (const correction of ['> This selection is withdrawn.', 'Historical note: "This selection is withdrawn."']) { + const p = input(); announcement(p).text += '\n' + correction; expect(verdict(p)).toBe(true); + } + const p = input(), m = announcement(p); + p.transcript.assistantMessages.push({ ...m, sessionId: 'foreign', text: 'This selection is withdrawn.' }); + expect(verdict(p)).toBe(true); +}); + +test('a later current withdrawal invalidates selection until a later reselection', () => { + const p = input(), m = announcement(p); + p.transcript.assistantMessages.push({ ...m, timestamp: new Date(Date.parse(m.timestamp) + 1000).toISOString(), text: 'This selection is withdrawn.' }); + expect(verdict(p)).toBe(false); + p.transcript.assistantMessages.push({ ...m, timestamp: new Date(Date.parse(m.timestamp) + 2000).toISOString() }); + expect(verdict(p)).toBe(true); +}); + +test('both regression sources select the same five existing scope observers', () => { + const expected = selectTests(['test/helpers/plan-scope-selection.ts'], E2E_TOUCHFILES, []).selected; + expect(expected).toHaveLength(5); + for (const path of ['test/design-scope-announcement-ao.test.ts', 'test/fixtures/design-scope-announcement-ao.json']) { + expect(selectTests([path], E2E_TOUCHFILES, []).selected).toEqual(expected); + } +}); diff --git a/test/dx-manual-handoff-ao.test.ts b/test/dx-manual-handoff-ao.test.ts new file mode 100644 index 000000000..40f6589a6 --- /dev/null +++ b/test/dx-manual-handoff-ao.test.ts @@ -0,0 +1,107 @@ +import {describe,expect,test} from 'bun:test'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import {hasNativePlanTerminal, classifyPlanCountFrame} from './helpers/claude-pty-runner'; +import type {NativePlanQuestionCall,PlanCountTranscript} from './helpers/plan-count-transcript'; +import captured from './fixtures/dx-manual-handoff-ao.json'; +import {E2E_TOUCHFILES,LLM_JUDGE_TOUCHFILES,GLOBAL_TOUCHFILES} from './helpers/touchfiles-data'; + +type Edit=(calls:NativePlanQuestionCall[], transcript:PlanCountTranscript, report:string)=>void; +function replay(edit?:Edit){ + const dir=fs.mkdtempSync(path.join(os.tmpdir(),'dx-manual-handoff-ao-')); + try { + const report=path.join(dir,'report.md');fs.writeFileSync(report,captured.reportContent); + const written=captured.provenance.reportMtimeMs/1000;fs.utimesSync(report,written,written); + const transcript={status:'ready',calls:structuredClone(captured.calls),assistantMessages:[],planReadyRequests:structuredClone(captured.planReadyRequests)} as PlanCountTranscript; + edit?.(transcript.calls,transcript,report); + return hasNativePlanTerminal(transcript,report,captured.provenance.startedAt,'plan_ready'); + } finally {fs.rmSync(dir,{recursive:true,force:true});} +} +function change(call:NativePlanQuestionCall,from:string,to:string){ + const q=call.questions[0]!;expect(q.question).toContain(from); + const selected=call.answers![q.question];q.question=q.question.replace(from,to);call.answers={[q.question]:selected!}; +} +describe('AO completed manual DX handoff preserves report freshness',()=>{ + test('shared completion callers register the regression with dense literal paths',()=>{ + for(const owner of ['plan-ceo-finding-count','plan-design-finding-count','plan-eng-finding-count','plan-devex-finding-count']){ + expect(E2E_TOUCHFILES[owner]).toContain('test/dx-manual-handoff-ao.test.ts'); + expect(E2E_TOUCHFILES[owner]).toContain('test/fixtures/dx-manual-handoff-ao.json'); + } + const arrays=[...Object.values(E2E_TOUCHFILES),...Object.values(LLM_JUDGE_TOUCHFILES),GLOBAL_TOUCHFILES]; + expect(arrays).toHaveLength(210); + for(const values of arrays)for(let i=0;i{ + expect(captured.calls).toHaveLength(2);expect(captured.events).toHaveLength(4); + expect(Date.parse(captured.calls[0]!.answeredAt!)).toBeLessThan(captured.provenance.reportMtimeMs); + expect(Date.parse(captured.calls[1]!.answeredAt!)).toBeGreaterThan(captured.provenance.reportMtimeMs); + expect(classifyPlanCountFrame(captured.screen)).toBe('plan_ready'); + expect(replay()).toBe(true); + }); + test('equivalent completed recap and manual roles retain current authority',()=>{ + for(const [from,to] of [ + [' (5/10 -> 8.5/10)',''], + ['5/10 -> 8.5/10','6/10 → 9/10'], + ['What should happen next?',"What's next?"], + ['The DX review found','The DX review identified'], + ['All are written into the plan as tasks T1 to T9.','All DX decisions and tasks are recorded in the plan.'], + ])expect(replay(calls=>change(calls[1]!,from!,to!)),to).toBe(true); + expect(replay(calls=>calls[1]!.questions[0]!.options.reverse())).toBe(true); + expect(replay(calls=>{const c=calls[1]!;change(c,'Net: hand off now as you asked, or chain the eng review here.','Net: hand off now as you asked, or chain the eng review here.\n> Historical example: add a new task before leaving.');})).toBe(true); + expect(replay(calls=>{const o=calls[1]!.questions[0]!.options[0]!;o.description=o.description!.replace('Plan exits now with all DX decisions and tasks recorded; nothing else is started.','Exit the plan now with all DX tasks and decisions recorded. No further review is started.');})).toBe(true); + }); + test('source, conditional or withdrawn completion facts cannot make a stale report current',()=>{ + const edits:Array<[string,string]>=[ + ['D13 — DX review','Source: D13 — DX review'], + ['DX review complete','DX review is not complete'], + ['DX review complete','DX review complete only after another decision'], + ['ELI10: The DX review found','ELI10: Earlier review assessment: The DX review found'], + ['ELI10: The DX review found','ELI10: If approved, the DX review found'], + ['All are written into the plan as tasks T1 to T9.','Example: All are written into the plan as tasks T1 to T9.'], + ['All are written into the plan as tasks T1 to T9.','Previously, all are written into the plan as tasks T1 to T9.'], + ['All are written into the plan as tasks T1 to T9.','"All are written into the plan as tasks T1 to T9."'], + ['All are written into the plan as tasks T1 to T9.','All will be written into the plan as tasks T1 to T9.'], + ['Project/branch/task:','Source:\nProject/branch/task:'], + ['Project/branch/task: ','Project/branch/task: If approved, '], + ['Project/branch/task: ','Project/branch/task: Source excerpt, not a current assessment: '], + ]; + for(const [from,to] of edits)expect(replay(calls=>change(calls[1]!,from,to)),to).toBe(false); + for(const suffix of [' This review is not complete.',' These tasks are not recorded.',' This review is "withdrawn".',' One DX decision remains unresolved.',' We must fix another issue.',' Add another migration task.',' Should we approve another change?',' ']) { + expect(replay(calls=>{const c=calls[1]!;change(c,c.questions[0]!.question,c.questions[0]!.question+suffix);}),suffix).toBe(false); + } + }); + test('selected manual action must close with recorded decisions and no new work',()=>{ + for(const prefix of ['Source: ','Earlier review assessment: ','If approved, ','> '])expect(replay(calls=>{const o=calls[1]!.questions[0]!.options[0]!;o.description=prefix+o.description;}),prefix).toBe(false); + for(const suffix of [' Also update the plan before exit.',' Run /plan-eng-review now.',' This plan is not complete.',' The tasks are "withdrawn".',' This manual handoff is cancelled.'])expect(replay(calls=>{calls[1]!.questions[0]!.options[0]!.description+=suffix;}),suffix).toBe(false); + for(const from of ['all DX decisions and tasks recorded','nothing else is started'])expect(replay(calls=>{const o=calls[1]!.questions[0]!.options[0]!;o.description=o.description!.replace(from,'more work remains');}),from).toBe(false); + for(const index of [1,2])expect(replay(calls=>{const c=calls[1]!,q=c.questions[0]!;c.answers={[q.question]:q.options[index]!.label};})).toBe(false); + }); + test('completed owned native answer identity remains mandatory',()=>{ + const mutations:Array<(c:NativePlanQuestionCall)=>void>=[ + c=>{c.answered=false;},c=>{c.failed=true;},c=>{c.sessionId='foreign';},c=>{c.toolUseId='';}, + c=>{c.answeredAt='invalid';},c=>{c.answeredAt=new Date(Date.now()+60_000).toISOString();}, + c=>{c.unansweredQuestionIndices=[0];},c=>{delete c.unansweredQuestionIndices;}, + c=>{c.questions[0]!.multiSelect=true;},c=>{c.questions[0]!.header='Issue decision';}, + c=>{c.questions.push(structuredClone(c.questions[0]!));}, + c=>{c.answers={wrong:c.questions[0]!.options[0]!.label};}, + c=>{c.answers![c.questions[0]!.question]='Not an offered answer';}, + c=>{c.answers!.extra='foreign';}, + c=>{c.questions[0]!.options[1]!.label=c.questions[0]!.options[0]!.label;}, + ]; + for(const edit of mutations)expect(replay(calls=>edit(calls[1]!)),edit.toString()).toBe(false); + }); + test('other modifying answers and complete report/current Exit gates remain unchanged',()=>{ + expect(replay(calls=>{calls[0]!.answeredAt=new Date(captured.provenance.reportMtimeMs+1).toISOString();})).toBe(false); + expect(replay((_calls,_t,report)=>{const time=Date.parse(captured.calls[0]!.answeredAt!)/1000-1;fs.utimesSync(report,time,time);})).toBe(false); + expect(replay((_calls,_t,report)=>fs.writeFileSync(report,'# Completion summary\nDone.'))).toBe(false); + expect(replay((_calls,_t,report)=>fs.unlinkSync(report))).toBe(false); + for(const mutate of [ + (t:PlanCountTranscript)=>{t.planReadyRequests=[];}, + (t:PlanCountTranscript)=>{t.planReadyRequests![0]!.failed=true;}, + (t:PlanCountTranscript)=>{t.planReadyRequests![0]!.sessionId='foreign';}, + (t:PlanCountTranscript)=>{t.planReadyRequests![0]!.timestamp=captured.calls[1]!.answeredAt!;}, + (t:PlanCountTranscript)=>{t.status='missing';}, + ])expect(replay((_calls,t)=>mutate(t))).toBe(false); + }); +}); diff --git a/test/fixtures/autoplan-final-gate-ao.json b/test/fixtures/autoplan-final-gate-ao.json new file mode 100644 index 000000000..4950d18d4 --- /dev/null +++ b/test/fixtures/autoplan-final-gate-ao.json @@ -0,0 +1,322 @@ +{ + "source": { + "proof": ".context/ship-source-ao-delta-paid-20260910-v1/ap-final-gate-boundary-plan-v1/proof.json", + "proofSHA256": "84ba8ef415caa29f72cfe48e72e9ee71346aff7ba51d605bdc3a1e3622975efa", + "publicSHA256": "49df000a7ac8251c7b56438baec010308f403388a2d3082759e525f3f93f3091", + "screenSHA256": "b22329efffdf54e9648a62d7e45595c6124498abca3ddae8fe9eef772c78982a" + }, + "cwd": "/tmp/gstack-paid-shard-6rS7wY/tmp/gstack-autoplan-chain-yq1CGO", + "commandStartedAt": 1789040142580, + "lowerBoundNote": "Owned parent launch lower bound from retained proof; exact slash-command time unavailable.", + "observedAt": 1789043120577, + "screen": "/tmp/gstack-paid-shard-6rS7wY/tmp/gstack-hermetic-2826645-EaQl2D/with-skills/.claude/plans/velvety-kindling-pretzel.md\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n \u2610 Approval\n\n\u2502 D2 \u2014 Final Approval Gate: approve the reviewed plan?\n\u2502 Project/branch/task: gstack-autoplan-chain-yq1CGO on main, /autoplan review of the User Dashboard plan.\n\u2502 ELI10: Four review passes (strategy, design, developer experience, engineering) ran with auto-decisions and turned\n\u2502 your one-page plan into a full spec with 69 logged decisions and 41 tasks. Your scope was kept intact. Six calls were\n\u2502 close enough that reasonable people could pick differently (listed above as T1-T6), and one reviewer wants to drop the\n\u2502 confirmation modal (C1). Approving locks the recommendations; overriding changes specific calls; revising re-runs the\n\u2502 affected phases.\n\u2502 Stakes if we pick wrong: approving with a call you disagree with means implementing something you will later rip out;\n\u2502 revising costs another review cycle (max 3).\n\u2502 Recommendation: A because every taste call defaults to your stated direction or the more complete option, and C1 keeps\n\u2502 your modal unless you say otherwise.\n\u2502 Note: options differ in kind, not coverage \u2014 no completeness score.\n\u2502 Net: approve now and adjust T1-T6 or C1 in code review, or spend one more cycle changing them in the plan first.\n\n\u276f 1. Approve as-is (recommended)\n \u2705 All six taste calls and C1 stay on your stated direction or the complete option; plan is ready for E0 and\n /ship.\ufffd\u2705 Writes review logs so the /ship dashboard shows CEO, design, DX and eng as reviewed via /autoplan.\ufffd\u274c Any\n disagreement with T1-T6 or C1 has to be raised later, in the consolidated spec or in code review.\n 2. Approve with overrides or challenge responses\n \u2705 Pick which of T1-T6 to flip, or accept C1 (drop the modal); I amend the plan and re-run Eng on the amended\n plan.\ufffd\u2705 Keeps everything else approved; counts as one of the three revision cycles.\ufffd\u274c Flipping T1 (three panel\n calls) or T2 (defer ActivityFeed) reshapes several accepted blocks and takes a full Eng re-run.\n 3. Interrogate a decision first\n \u2705 Ask about any of the 69 decisions, a finding, or a subagent's reasoning before deciding; I answer and re-present\n this gate.\ufffd\u2705 No plan changes until you choose; nothing is lost.\ufffd\u274c Delays approval by one round trip.\n 4. Revise the plan or reject\n \u2705 Revise: name what changes (scope -> CEO rerun, UI -> design, DX, tests or architecture -> eng); Eng always\n re-runs last. Reject: start over from the restore point.\ufffd\u2705 The restore point holds the original plan bytes, so\n rejection is safe.\ufffd\u274c A revise cycle re-dispatches the affected reviewers and takes roughly as long as the phase\n did the first time.\n 5. Type something.\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n 6. Chat about this\n\nEnter to select \u00b7 \u2191/\u2193 to navigate \u00b7 Esc to cancel\n", + "call": { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "toolUseId": "toolu_01WyqMZCRbD4LvJX2AdyF2qq", + "questions": [ + { + "question": "D2 \u2014 Final Approval Gate: approve the reviewed plan?\nProject/branch/task: gstack-autoplan-chain-yq1CGO on main, /autoplan review of the User Dashboard plan.\nELI10: Four review passes (strategy, design, developer experience, engineering) ran with auto-decisions and turned your one-page plan into a full spec with 69 logged decisions and 41 tasks. Your scope was kept intact. Six calls were close enough that reasonable people could pick differently (listed above as T1-T6), and one reviewer wants to drop the confirmation modal (C1). Approving locks the recommendations; overriding changes specific calls; revising re-runs the affected phases.\nStakes if we pick wrong: approving with a call you disagree with means implementing something you will later rip out; revising costs another review cycle (max 3).\nRecommendation: A because every taste call defaults to your stated direction or the more complete option, and C1 keeps your modal unless you say otherwise.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: approve now and adjust T1-T6 or C1 in code review, or spend one more cycle changing them in the plan first.", + "header": "Approval", + "multiSelect": false, + "options": [ + { + "label": "Approve as-is (recommended)", + "description": "\u2705 All six taste calls and C1 stay on your stated direction or the complete option; plan is ready for E0 and /ship.\n\u2705 Writes review logs so the /ship dashboard shows CEO, design, DX and eng as reviewed via /autoplan.\n\u274c Any disagreement with T1-T6 or C1 has to be raised later, in the consolidated spec or in code review." + }, + { + "label": "Approve with overrides or challenge responses", + "description": "\u2705 Pick which of T1-T6 to flip, or accept C1 (drop the modal); I amend the plan and re-run Eng on the amended plan.\n\u2705 Keeps everything else approved; counts as one of the three revision cycles.\n\u274c Flipping T1 (three panel calls) or T2 (defer ActivityFeed) reshapes several accepted blocks and takes a full Eng re-run." + }, + { + "label": "Interrogate a decision first", + "description": "\u2705 Ask about any of the 69 decisions, a finding, or a subagent's reasoning before deciding; I answer and re-present this gate.\n\u2705 No plan changes until you choose; nothing is lost.\n\u274c Delays approval by one round trip." + }, + { + "label": "Revise the plan or reject", + "description": "\u2705 Revise: name what changes (scope -> CEO rerun, UI -> design, DX, tests or architecture -> eng); Eng always re-runs last. Reject: start over from the restore point.\n\u2705 The restore point holds the original plan bytes, so rejection is safe.\n\u274c A revise cycle re-dispatches the affected reviewers and takes roughly as long as the phase did the first time." + } + ] + } + ], + "answered": false, + "failed": false + }, + "gateUse": { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "timestamp": "2026-09-10T12:22:06.810Z", + "toolUseId": "toolu_01WyqMZCRbD4LvJX2AdyF2qq", + "kind": "use", + "name": "AskUserQuestion", + "input": { + "questions": [ + { + "question": "D2 \u2014 Final Approval Gate: approve the reviewed plan?\nProject/branch/task: gstack-autoplan-chain-yq1CGO on main, /autoplan review of the User Dashboard plan.\nELI10: Four review passes (strategy, design, developer experience, engineering) ran with auto-decisions and turned your one-page plan into a full spec with 69 logged decisions and 41 tasks. Your scope was kept intact. Six calls were close enough that reasonable people could pick differently (listed above as T1-T6), and one reviewer wants to drop the confirmation modal (C1). Approving locks the recommendations; overriding changes specific calls; revising re-runs the affected phases.\nStakes if we pick wrong: approving with a call you disagree with means implementing something you will later rip out; revising costs another review cycle (max 3).\nRecommendation: A because every taste call defaults to your stated direction or the more complete option, and C1 keeps your modal unless you say otherwise.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: approve now and adjust T1-T6 or C1 in code review, or spend one more cycle changing them in the plan first.", + "header": "Approval", + "multiSelect": false, + "options": [ + { + "label": "Approve as-is (recommended)", + "description": "\u2705 All six taste calls and C1 stay on your stated direction or the complete option; plan is ready for E0 and /ship.\n\u2705 Writes review logs so the /ship dashboard shows CEO, design, DX and eng as reviewed via /autoplan.\n\u274c Any disagreement with T1-T6 or C1 has to be raised later, in the consolidated spec or in code review." + }, + { + "label": "Approve with overrides or challenge responses", + "description": "\u2705 Pick which of T1-T6 to flip, or accept C1 (drop the modal); I amend the plan and re-run Eng on the amended plan.\n\u2705 Keeps everything else approved; counts as one of the three revision cycles.\n\u274c Flipping T1 (three panel calls) or T2 (defer ActivityFeed) reshapes several accepted blocks and takes a full Eng re-run." + }, + { + "label": "Interrogate a decision first", + "description": "\u2705 Ask about any of the 69 decisions, a finding, or a subagent's reasoning before deciding; I answer and re-present this gate.\n\u2705 No plan changes until you choose; nothing is lost.\n\u274c Delays approval by one round trip." + }, + { + "label": "Revise the plan or reject", + "description": "\u2705 Revise: name what changes (scope -> CEO rerun, UI -> design, DX, tests or architecture -> eng); Eng always re-runs last. Reject: start over from the restore point.\n\u2705 The restore point holds the original plan bytes, so rejection is safe.\n\u274c A revise cycle re-dispatches the affected reviewers and takes roughly as long as the phase did the first time." + } + ] + } + ] + }, + "messageId": "msg_011CeukPbjmuqmh8XYNSdXeU", + "requestId": "req_011CeukPaZpPTJpJLbrjgz73" + }, + "transcript": { + "status": "ready", + "calls": [ + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "toolUseId": "toolu_01Ud1CJ8i13mtFvQyy7aLWZs", + "questions": [ + { + "question": "D1 \u2014 Add gstack skill routing rules to this project's CLAUDE.md?\nProject/branch/task: fixture repo on main, running /autoplan.\nELI10: gstack works best when your project's CLAUDE.md includes skill routing rules. These rules tell Claude which skill to invoke when you ask for things like bug hunting, shipping, or plan review, so you don't have to remember slash-command names. Without them, skills only run when you type them by hand.\nStakes if we pick wrong: Skipping means more manual invocation; adding means one extra committed section in CLAUDE.md.\nRecommendation: A because routing makes the rest of gstack discoverable with zero ongoing cost.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: convenience now vs. keeping CLAUDE.md untouched. This only happens once per project.", + "header": "Routing", + "multiSelect": false, + "options": [ + { + "label": "Add routing rules to CLAUDE.md (recommended)", + "description": "\u2705 Natural-language requests auto-route to the right gstack skill.\n\u2705 Teammates using the repo get the same routing behavior.\n\u274c Adds a committed section to CLAUDE.md (created if missing)." + }, + { + "label": "No thanks, I'll invoke skills manually", + "description": "\u2705 CLAUDE.md stays exactly as it is, no new commit.\n\u2705 You keep full control over when skills run.\n\u274c Skills only fire when you type the slash command yourself." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D1 \u2014 Add gstack skill routing rules to this project's CLAUDE.md?\nProject/branch/task: fixture repo on main, running /autoplan.\nELI10: gstack works best when your project's CLAUDE.md includes skill routing rules. These rules tell Claude which skill to invoke when you ask for things like bug hunting, shipping, or plan review, so you don't have to remember slash-command names. Without them, skills only run when you type them by hand.\nStakes if we pick wrong: Skipping means more manual invocation; adding means one extra committed section in CLAUDE.md.\nRecommendation: A because routing makes the rest of gstack discoverable with zero ongoing cost.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: convenience now vs. keeping CLAUDE.md untouched. This only happens once per project.": "Add routing rules to CLAUDE.md (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-10T11:36:36.415Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "toolUseId": "toolu_01WyqMZCRbD4LvJX2AdyF2qq", + "questions": [ + { + "question": "D2 \u2014 Final Approval Gate: approve the reviewed plan?\nProject/branch/task: gstack-autoplan-chain-yq1CGO on main, /autoplan review of the User Dashboard plan.\nELI10: Four review passes (strategy, design, developer experience, engineering) ran with auto-decisions and turned your one-page plan into a full spec with 69 logged decisions and 41 tasks. Your scope was kept intact. Six calls were close enough that reasonable people could pick differently (listed above as T1-T6), and one reviewer wants to drop the confirmation modal (C1). Approving locks the recommendations; overriding changes specific calls; revising re-runs the affected phases.\nStakes if we pick wrong: approving with a call you disagree with means implementing something you will later rip out; revising costs another review cycle (max 3).\nRecommendation: A because every taste call defaults to your stated direction or the more complete option, and C1 keeps your modal unless you say otherwise.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: approve now and adjust T1-T6 or C1 in code review, or spend one more cycle changing them in the plan first.", + "header": "Approval", + "multiSelect": false, + "options": [ + { + "label": "Approve as-is (recommended)", + "description": "\u2705 All six taste calls and C1 stay on your stated direction or the complete option; plan is ready for E0 and /ship.\n\u2705 Writes review logs so the /ship dashboard shows CEO, design, DX and eng as reviewed via /autoplan.\n\u274c Any disagreement with T1-T6 or C1 has to be raised later, in the consolidated spec or in code review." + }, + { + "label": "Approve with overrides or challenge responses", + "description": "\u2705 Pick which of T1-T6 to flip, or accept C1 (drop the modal); I amend the plan and re-run Eng on the amended plan.\n\u2705 Keeps everything else approved; counts as one of the three revision cycles.\n\u274c Flipping T1 (three panel calls) or T2 (defer ActivityFeed) reshapes several accepted blocks and takes a full Eng re-run." + }, + { + "label": "Interrogate a decision first", + "description": "\u2705 Ask about any of the 69 decisions, a finding, or a subagent's reasoning before deciding; I answer and re-present this gate.\n\u2705 No plan changes until you choose; nothing is lost.\n\u274c Delays approval by one round trip." + }, + { + "label": "Revise the plan or reject", + "description": "\u2705 Revise: name what changes (scope -> CEO rerun, UI -> design, DX, tests or architecture -> eng); Eng always re-runs last. Reject: start over from the restore point.\n\u2705 The restore point holds the original plan bytes, so rejection is safe.\n\u274c A revise cycle re-dispatches the affected reviewers and takes roughly as long as the phase did the first time." + } + ] + } + ], + "answered": false, + "failed": false + } + ], + "assistantMessages": [ + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "I'll start the /autoplan pipeline. First, the preamble to get session status.", + "timestamp": "2026-09-10T11:35:57.737Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The preamble fired a one-time routing-injection instruction. I'll ask that question and pull repo context in parallel.", + "timestamp": "2026-09-10T11:36:19.372Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "I've completed phase 0 intake: using `ui-heavy-feature.md` as the source plan, `main` as the base branch (no remote), and only the Claude adversarial subagent as the outside voice since Codex reviews are disabled. Now setting up the restore point and snapshot.\n\n", + "timestamp": "2026-09-10T11:37:28.689Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Phase 0 confirms UI and DX are in scope, no design doc exists, and Codex is disabled per config. Moving into Phase 1 (CEO), starting with binding the methodology.", + "timestamp": "2026-09-10T11:37:54.148Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Now I read the CEO methodology in all four required ranges.", + "timestamp": "2026-09-10T11:38:03.152Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "All four CEO methodology ranges read through EOF (lines 1-2260). Now binding the CEO phase input snapshot and running the pre-review system audit.", + "timestamp": "2026-09-10T11:38:20.183Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The repo only has README.md and the plan file, so I can't verify the plan's referenced contracts against actual code \u2014 I'll treat them as stated dependencies and flag that at the gate. Now dispatching the independent Claude CEO subagent with the snapshot tool's prompt.\n\n", + "timestamp": "2026-09-10T11:38:45.898Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Waiting for the independent CEO reviewer to finish. Per the phase's completion barrier, no further review work until its result lands.", + "timestamp": "2026-09-10T11:38:56.493Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The CEO subagent completed with a matching input hash and 13 findings (3 high, 10 medium, 0 critical). Codex is disabled, so the outside cell is N/A. Now I run the full primary CEO review and write every required output into the plan file's Review record.", + "timestamp": "2026-09-10T11:41:17.126Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Next I need the CEO `amend` check on the plan file and the adversarial spec review of the CEO plan archive. They are independent, so both go now.", + "timestamp": "2026-09-10T11:46:45.187Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The amend check passed and reorganized the plan so the accepted block now sits in the Implementation plan for later phases to see. I'll read the file back in full as required while the spec reviewer runs.\n\n", + "timestamp": "2026-09-10T11:47:04.940Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Plan file readback confirms structure: the accepted CEO block now sits in the Implementation plan for downstream reviewers, and the full Review record with all 21 audit rows is intact. The only open item before closing Phase 1 is the adversarial spec review of the CEO plan archive, which is still running. Waiting for it.", + "timestamp": "2026-09-10T11:47:23.788Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The spec reviewer scored the CEO archive 4/10 with 23 issues. About half come from the archive lacking the base plan context (by design it is a decision log), but several are real: no per-source timeout on the aggregate, persistent error toasts can starve the queue, one live region for mixed severities, `returnTo` validation rule unstated, quick actions not enumerated, provider mounted app-wide. I'll fix the archive, record the refinements in the plan's Review record for the design and eng phases to absorb, and re-dispatch the reviewer (iteration 2 of 3).", + "timestamp": "2026-09-10T11:48:40.613Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "CEO plan archive rewritten with a baseline scope section and concrete definitions. Refinements R1 through R8 recorded in the plan for the design and eng phases to absorb. Spec review iteration 2 is running; nothing else in Phase 1 can proceed until it returns.", + "timestamp": "2026-09-10T11:50:00.087Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Iteration 2 scored 7/10 with 14 issues, most of them precise definitional gaps (closed error-code set, the N in the toast copy, zero-unread detection with more pages, alert windows, the join key for the metric, timeout not cancelling queries). I'll fix all of them, simplify the toast stack to one visible toast per region, and run iteration 3 (the last permitted).", + "timestamp": "2026-09-10T11:51:39.833Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Iteration 3 (the final permitted round) is running against the updated archive. Whatever it returns closes the loop: remaining issues get recorded as Reviewer Concerns, then Phase 1 closes and Phase 2 (design) begins.", + "timestamp": "2026-09-10T11:52:41.335Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Iteration 3 scored 7/10 again with 18 new, finer-grained issues. That is the cap, so the loop stops. I'll decide each item with the principles, persist them as Reviewer Concerns in the archive and as refinements in the plan, log the spec-review metrics, and close Phase 1.", + "timestamp": "2026-09-10T11:54:47.613Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Phase 1 finished: Codex disabled, Claude subagent found 13 issues (none critical), with 3 single-voice disagreements queued for the gate. Moving into Phase 2, a design review since UI scope was detected, and loading its methodology now.\n\n", + "timestamp": "2026-09-10T11:55:32.012Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Waiting for the independent design reviewer. Per the completion barrier, no further design review work runs until its result arrives.", + "timestamp": "2026-09-10T11:56:13.689Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The design subagent completed with a matching input hash: 14 findings (3 critical: panel order, read/unread model, toast count source), plus 12 ranked ambiguities. Codex is disabled, so the outside column is N/A. Now the full primary design review: Step 0 rating, litmus scorecard, all 7 passes, and the amendments written into the plan.", + "timestamp": "2026-09-10T11:59:30.380Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The probe confirms no OpenAI API key is available, so mockups can't be generated this run \u2014 I'll note that and move on to the text review, now writing the Phase 2 outputs.\n\n", + "timestamp": "2026-09-10T12:00:46.401Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Next I'll design the audit-trail rows edit, followed by the amend check\u2014running the edit first since amend depends on it landing beforehand.\n\n", + "timestamp": "2026-09-10T12:03:13.177Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Phase 2 is done: Codex disabled, the Claude subagent found 14 issues, and taste item T6 is queued. Next I'll pull the readback of the moved block, the DX phase section, and the DX methodology binding, which can all be done independently.\n\n", + "timestamp": "2026-09-10T12:03:43.147Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Confirmed the design block is now correctly placed after the CEO block. Moving into Phase 2.5 (DX): I'll read the DX methodology across all four ranges and create the DX snapshot, which can be done independently.\n\n", + "timestamp": "2026-09-10T12:04:00.674Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "DX methodology read through EOF (2115 lines), snapshot created. Dispatching the independent DX subagent with the tool's prompt verbatim, as the only call this turn.", + "timestamp": "2026-09-10T12:04:19.052Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Waiting for the independent DX reviewer. Per the completion barrier, no further DX review work runs until its result arrives.", + "timestamp": "2026-09-10T12:04:34.291Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The DX subagent completed with a matching hash: 21 findings, one critical (the response `data` shape is stated two ways: bare array vs `{items, hasMore}`), six high. It framed the developer as the implementing and maintaining engineer, which is the right lens for an internal feature. Now the full primary DX review and its outputs.", + "timestamp": "2026-09-10T12:07:49.173Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Phase 2.5 is complete\u2014DX amend passed, all pre-Phase 3 checks are in place\u2014so I'm moving on to Phase 3 (Eng), the required gate.\n\n", + "timestamp": "2026-09-10T12:11:24.933Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Readback confirms the DX block landed. Reading the eng methodology in all three ranges and creating the eng snapshot in parallel.", + "timestamp": "2026-09-10T12:11:37.488Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Waiting for the independent eng reviewer. Per the completion barrier, no eng review work runs until its result lands.", + "timestamp": "2026-09-10T12:12:10.633Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The eng subagent completed with a matching hash: 30 findings across architecture, edge cases, tests, security and hidden complexity, with five flagged as resolve-before-implementation (mandatory query cancellation, flag-off API behavior, authorization tests, no-store cache header, two spec conflicts). Now the full primary eng review, the test plan artifact, and the eng accepted block.", + "timestamp": "2026-09-10T12:16:00.959Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Eng audit-trail rows, the eng `amend` check, and the Phase 4 aggregator section are the remaining items; the read is independent of the two writes.", + "timestamp": "2026-09-10T12:19:54.219Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Pre-gate verification passes: every required output is present in the plan (premise challenge, error and rescue registry, failure modes registries, NOT-in-scope and what-exists sections for all four phases, dream state delta, consensus tables, litmus scorecard, DX scorecard and checklist, journey map, empathy narrative, TTHW, architecture diagram, test diagram, test plan artifact, cross-phase themes) and the audit trail holds 69 decision rows. Aggregation produced 41 tasks across four phases. Presenting the Final Approval Gate.\n\n## /autoplan Review Complete\n\n### Plan Summary\nA new post-login `/dashboard` with three panels (quick actions, notifications, activity), one aggregate endpoint with a per-key result envelope, a confirmation dialog for mark-all-as-read, and a toast system. The review kept the user's scope intact and made it a normative spec: partial-failure contract, cancellation, panel states and copy, instrumentation, rollout with kill criterion, shared types/config/copy/fixtures, and 47 planned test paths with zero critical failure-mode gaps.\n\n### Decisions Made: 69 total (62 auto-decided, 6 taste choices, 1 user challenge)\n\n### User Challenges\n\n**Challenge C1: confirmation modal for \"Mark all as read\"** (from CEO phase; single voice, Codex disabled)\nYou said: a modal dialog confirms \"Mark all as read\".\nThe Claude subagent recommends: drop the modal; use a single button with inline state.\nWhy: the action is idempotent and snapshot-bounded, so a confirmation reads as friction on a low-stakes action.\nWhat we might be missing: there is no \"mark unread\" API, so the action is not reversible for the member, and the team may have chosen the modal from support history.\nIf we're wrong, the cost is: members lose a safety step before an irreversible bulk change, and a one-line UI change later.\nYour call. Your original direction stands unless you change it. The modal's copy, in-flight behavior and focus rules are fully specified either way.\n\n### Your Choices (taste decisions)\n\n**T1: aggregate endpoint vs three parallel panel calls** (CEO). I recommend the aggregate endpoint with a per-key envelope (completeness 9/10, one round trip, one latency budget). Three panel calls (7/10) would need a new eligibility endpoint anyway and triple the instrumentation points.\n**T2: keep ActivityFeed in v1** (CEO). I recommend keeping it, per your direction; the content rule and kill criterion make it removable with data. Deferring it would shrink v1 to two panels and a smaller first release.\n**T3: toast system vs inline feedback** (CEO). I recommend the toast system, because policy requires a live region and nothing exists today. Inline-only feedback avoids a new provider but leaves the live-region requirement unmet.\n**T4: keyboard shortcuts for quick actions deferred** (CEO). I recommend deferring to TODOS; the accessibility announcement design is not settled. Including them adds two or three files and conflict risk.\n**T5: `returnTo` deep-link preservation touches login code** (CEO spec review). I recommend keeping it; it is the one accepted item outside the dashboard blast radius and it fixes a real deep-link regression. Dropping it means every signed-out deep link lands on the dashboard instead of the intended page.\n**T6: \"Resume assigned work\" as the primary button** (Design). I recommend making it primary; it is the task-start action the metric depends on and this is static hierarchy, not personalization. Equal-weight buttons keep the panel neutral but slow the first scan.\n\n### Auto-Decided: 62 decisions (see Decision Audit Trail in the plan file)\n\n### Review Scores\n- CEO: SELECTIVE EXPANSION; approach A; 15 expansions proposed, 9 accepted, 4 deferred, 2 skipped; 0 critical gaps. Spec-review loop 3 iterations, score 4 to 7/10, remaining items resolved by decision.\n- CEO Voices: Codex disabled; Claude subagent 13 issues (0 critical); Consensus N/A (outside disabled).\n- Design: 4/10 to 8/10; Pass 5 stays at 7 because no DESIGN.md exists; 14 decisions added.\n- Design Voices: Codex disabled; Claude subagent 14 issues (3 critical); Consensus N/A.\n- DX: 4.5/10 to 8/10; TTHW from over 10 minutes to about 3 minutes via a fixtures module and a dev-only state switch.\n- DX Voices: Codex disabled; Claude subagent 21 issues (1 critical); Consensus N/A.\n- Eng: 17 issues across architecture, quality and performance plus 8 test findings, all folded in; 47 planned test paths; 0 critical failure-mode gaps; 5 parallel lanes.\n- Eng Voices: Codex disabled; Claude subagent 30 issues (4 high); Consensus N/A.\n\nOutside coverage is disabled for all four phases by config (`codex_reviews=disabled`). All outside voices were same-harness Claude subagents with fresh context.\n\n### Cross-Phase Themes\n- **Partial failure must never blank the page**: flagged independently in all four phases. Resolved by the envelope, 1 s deadline with cancellation, stale state and two-tier parsing.\n- **Spec drift across amendment blocks**: flagged by the CEO spec reviewer, DX and Eng. Resolved by the precedence rule and task E0 (consolidate before code).\n- **Mark-all-as-read semantics**: count, Escape, zero-unread and snapshot rules were questioned by at least two phases each. Resolved in the design, DX and eng blocks; the modal itself is C1 above.\n- **G1 contracts are load-bearing**: every phase added rows. Resolved by making G1 completion part of E0.\n\n### Gate items that are not decisions\n- **G1**: this fixture repo has no source, so every \"existing contract\" the plan names is unverified. The implementer confirms each one into CONTRACTS.md before component code; any missing item becomes new scope.\n- **Deferred writes** (plan mode forbids edits outside the plan file): the CLAUDE.md skill-routing section you approved at the start, and the TODOS.md entries below.\n- **Mockups**: the designer binary reports \"No OpenAI API key found.\" Run `$D setup`, then `/plan-design-review` or `/design-shotgun` for a comparison board. `/design-consultation` is recommended before implementation to record a DESIGN.md.\n\n### Deferred to TODOS.md\n1. Live polling or push refresh of notifications (new infra).\n2. Undo for mark-all-as-read (needs a mark-unread mutation API).\n3. Keyboard shortcuts for quick actions (taste T4).\n4. Prefetch dashboard payload during login redirect (login-flow infra).\n5. RTL layout and 200% zoom verification (design debt).\n6. Streaming or fastest-panel-first response (only if the 1 s deadline proves slow).\n7. Server-side dashboard cache (only if p95 misses budget).\n\n### Implementation Tasks (aggregated across phases)\n41 tasks, 33 P1 and 8 P2. Later-phase tasks supersede earlier ones where they overlap (for example D6/E6 refine T6, D7 refines T7, X1 fixes the envelope shape in T1); task E0 reconciles them into one spec first. Full list with \"Surfaced by\" and \"Files\" saved at `~/.gstack/projects/gstack-autoplan-chain-yq1CGO/aggregated-tasks-autoplan-20260910113655.md`.\n\n- **E0 (P1)** spec \u2014 consolidate accepted blocks into one normative section; complete G1 into CONTRACTS.md\n- **X1 (P1)** shared types \u2014 envelope, code unions, parseDashboardResponse\n- **X3 (P1)** config + copy \u2014 dashboardConfig, dashboardCopy, ToastProvider props\n- **X2 (P1)** fixtures + dev switch \u2014 named envelopes, `?__fixture=` in dev builds only\n- **T1 / X4 / E1 / E4 (P1)** endpoint \u2014 per-key envelope, 1 s deadline with cancellation, eligibility semantics, no-store, hasMore rule\n- **E2 (P1)** flag-off path \u2014 404 feature_disabled, client navigates, excluded from 403 alert\n- **T3 / X6 / E8 (P1)** login redirect \u2014 validated returnTo with the full bypass and loop test suite\n- **E3 (P1)** authorization tests\n- **T2 / D5 / E5 (P1)** PanelFrame \u2014 reducer with stale state, shaped skeletons, quiet errors, focus rules, ErrorBoundary\n- **D1 / D2 (P1)** layout and page header \u2014 panel order, landmarks, single Refresh with relative time\n- **D3 / D4 / D9 (P1, P2)** panels \u2014 notifications read model, quick actions, row anatomy\n- **T6 / D6 / E6 (P1)** MarkAllReadDialog \u2014 copy, in-flight rules, hasMore variant, CSRF loop guard, no auto-retry\n- **T7 / D7 (P1)** ToastProvider \u2014 two live regions, one visible per region, queues, placement\n- **D8 / X9 (P1, P2)** page-level states with requestId reference\n- **X5 (P1)** client retry semantics\n- **T4 / X7 (P1)** instrumentation with the event grammar\n- **T5 / E9 (P1)** perf, load profile, alerts, metrics, index check\n- **E7 (P1)** focus and copy fallbacks\n- **T10 (P1)** dependency check (G1)\n- **T8, T9, D10, E10, X8, X10 (P2)** panel headers, rollout notes, a11y floor, guards and lint, docs, dialog Escape prop", + "timestamp": "2026-09-10T12:21:51.719Z" + } + ] + }, + "hits": [ + { + "phase": 1, + "ts": 1789041332012 + }, + { + "phase": 2, + "ts": 1789041823147 + } + ], + "observedAtSource": "Retained boundary capture.at" +} diff --git a/test/fixtures/autoplan-phase-dash-ao.json b/test/fixtures/autoplan-phase-dash-ao.json new file mode 100644 index 000000000..cc56270b4 --- /dev/null +++ b/test/fixtures/autoplan-phase-dash-ao.json @@ -0,0 +1,190 @@ +{ + "source": { + "observation": ".context/ship-source-ao-delta-paid-20260910-v1/ap-final-gate-boundary-plan-v1/observation.json", + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "boundaryProof": "ap-final-gate-boundary-plan-v1/proof.json" + }, + "commandLowerBound": 1789040142580, + "message": { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Phase 2.5 is complete\u2014DX amend passed, all pre-Phase 3 checks are in place\u2014so I'm moving on to Phase 3 (Eng), the required gate.\n\n", + "timestamp": "2026-09-10T12:11:24.933Z" + }, + "orderedMessages": [ + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "I'll start the /autoplan pipeline. First, the preamble to get session status.", + "timestamp": "2026-09-10T11:35:57.737Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The preamble fired a one-time routing-injection instruction. I'll ask that question and pull repo context in parallel.", + "timestamp": "2026-09-10T11:36:19.372Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "I've completed phase 0 intake: using `ui-heavy-feature.md` as the source plan, `main` as the base branch (no remote), and only the Claude adversarial subagent as the outside voice since Codex reviews are disabled. Now setting up the restore point and snapshot.\n\n", + "timestamp": "2026-09-10T11:37:28.689Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Phase 0 confirms UI and DX are in scope, no design doc exists, and Codex is disabled per config. Moving into Phase 1 (CEO), starting with binding the methodology.", + "timestamp": "2026-09-10T11:37:54.148Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Now I read the CEO methodology in all four required ranges.", + "timestamp": "2026-09-10T11:38:03.152Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "All four CEO methodology ranges read through EOF (lines 1-2260). Now binding the CEO phase input snapshot and running the pre-review system audit.", + "timestamp": "2026-09-10T11:38:20.183Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The repo only has README.md and the plan file, so I can't verify the plan's referenced contracts against actual code \u2014 I'll treat them as stated dependencies and flag that at the gate. Now dispatching the independent Claude CEO subagent with the snapshot tool's prompt.\n\n", + "timestamp": "2026-09-10T11:38:45.898Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Waiting for the independent CEO reviewer to finish. Per the phase's completion barrier, no further review work until its result lands.", + "timestamp": "2026-09-10T11:38:56.493Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The CEO subagent completed with a matching input hash and 13 findings (3 high, 10 medium, 0 critical). Codex is disabled, so the outside cell is N/A. Now I run the full primary CEO review and write every required output into the plan file's Review record.", + "timestamp": "2026-09-10T11:41:17.126Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Next I need the CEO `amend` check on the plan file and the adversarial spec review of the CEO plan archive. They are independent, so both go now.", + "timestamp": "2026-09-10T11:46:45.187Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The amend check passed and reorganized the plan so the accepted block now sits in the Implementation plan for later phases to see. I'll read the file back in full as required while the spec reviewer runs.\n\n", + "timestamp": "2026-09-10T11:47:04.940Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Plan file readback confirms structure: the accepted CEO block now sits in the Implementation plan for downstream reviewers, and the full Review record with all 21 audit rows is intact. The only open item before closing Phase 1 is the adversarial spec review of the CEO plan archive, which is still running. Waiting for it.", + "timestamp": "2026-09-10T11:47:23.788Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The spec reviewer scored the CEO archive 4/10 with 23 issues. About half come from the archive lacking the base plan context (by design it is a decision log), but several are real: no per-source timeout on the aggregate, persistent error toasts can starve the queue, one live region for mixed severities, `returnTo` validation rule unstated, quick actions not enumerated, provider mounted app-wide. I'll fix the archive, record the refinements in the plan's Review record for the design and eng phases to absorb, and re-dispatch the reviewer (iteration 2 of 3).", + "timestamp": "2026-09-10T11:48:40.613Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "CEO plan archive rewritten with a baseline scope section and concrete definitions. Refinements R1 through R8 recorded in the plan for the design and eng phases to absorb. Spec review iteration 2 is running; nothing else in Phase 1 can proceed until it returns.", + "timestamp": "2026-09-10T11:50:00.087Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Iteration 2 scored 7/10 with 14 issues, most of them precise definitional gaps (closed error-code set, the N in the toast copy, zero-unread detection with more pages, alert windows, the join key for the metric, timeout not cancelling queries). I'll fix all of them, simplify the toast stack to one visible toast per region, and run iteration 3 (the last permitted).", + "timestamp": "2026-09-10T11:51:39.833Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Iteration 3 (the final permitted round) is running against the updated archive. Whatever it returns closes the loop: remaining issues get recorded as Reviewer Concerns, then Phase 1 closes and Phase 2 (design) begins.", + "timestamp": "2026-09-10T11:52:41.335Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Iteration 3 scored 7/10 again with 18 new, finer-grained issues. That is the cap, so the loop stops. I'll decide each item with the principles, persist them as Reviewer Concerns in the archive and as refinements in the plan, log the spec-review metrics, and close Phase 1.", + "timestamp": "2026-09-10T11:54:47.613Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Phase 1 finished: Codex disabled, Claude subagent found 13 issues (none critical), with 3 single-voice disagreements queued for the gate. Moving into Phase 2, a design review since UI scope was detected, and loading its methodology now.\n\n", + "timestamp": "2026-09-10T11:55:32.012Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Waiting for the independent design reviewer. Per the completion barrier, no further design review work runs until its result arrives.", + "timestamp": "2026-09-10T11:56:13.689Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The design subagent completed with a matching input hash: 14 findings (3 critical: panel order, read/unread model, toast count source), plus 12 ranked ambiguities. Codex is disabled, so the outside column is N/A. Now the full primary design review: Step 0 rating, litmus scorecard, all 7 passes, and the amendments written into the plan.", + "timestamp": "2026-09-10T11:59:30.380Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The probe confirms no OpenAI API key is available, so mockups can't be generated this run \u2014 I'll note that and move on to the text review, now writing the Phase 2 outputs.\n\n", + "timestamp": "2026-09-10T12:00:46.401Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Next I'll design the audit-trail rows edit, followed by the amend check\u2014running the edit first since amend depends on it landing beforehand.\n\n", + "timestamp": "2026-09-10T12:03:13.177Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Phase 2 is done: Codex disabled, the Claude subagent found 14 issues, and taste item T6 is queued. Next I'll pull the readback of the moved block, the DX phase section, and the DX methodology binding, which can all be done independently.\n\n", + "timestamp": "2026-09-10T12:03:43.147Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Confirmed the design block is now correctly placed after the CEO block. Moving into Phase 2.5 (DX): I'll read the DX methodology across all four ranges and create the DX snapshot, which can be done independently.\n\n", + "timestamp": "2026-09-10T12:04:00.674Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "DX methodology read through EOF (2115 lines), snapshot created. Dispatching the independent DX subagent with the tool's prompt verbatim, as the only call this turn.", + "timestamp": "2026-09-10T12:04:19.052Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Waiting for the independent DX reviewer. Per the completion barrier, no further DX review work runs until its result arrives.", + "timestamp": "2026-09-10T12:04:34.291Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The DX subagent completed with a matching hash: 21 findings, one critical (the response `data` shape is stated two ways: bare array vs `{items, hasMore}`), six high. It framed the developer as the implementing and maintaining engineer, which is the right lens for an internal feature. Now the full primary DX review and its outputs.", + "timestamp": "2026-09-10T12:07:49.173Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Phase 2.5 is complete\u2014DX amend passed, all pre-Phase 3 checks are in place\u2014so I'm moving on to Phase 3 (Eng), the required gate.\n\n", + "timestamp": "2026-09-10T12:11:24.933Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Readback confirms the DX block landed. Reading the eng methodology in all three ranges and creating the eng snapshot in parallel.", + "timestamp": "2026-09-10T12:11:37.488Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Waiting for the independent eng reviewer. Per the completion barrier, no eng review work runs until its result lands.", + "timestamp": "2026-09-10T12:12:10.633Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "The eng subagent completed with a matching hash: 30 findings across architecture, edge cases, tests, security and hidden complexity, with five flagged as resolve-before-implementation (mandatory query cancellation, flag-off API behavior, authorization tests, no-store cache header, two spec conflicts). Now the full primary eng review, the test plan artifact, and the eng accepted block.", + "timestamp": "2026-09-10T12:16:00.959Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Eng audit-trail rows, the eng `amend` check, and the Phase 4 aggregator section are the remaining items; the read is independent of the two writes.", + "timestamp": "2026-09-10T12:19:54.219Z" + }, + { + "sessionId": "4ef369ef-d229-4f75-869d-1182aa56cff5", + "text": "Pre-gate verification passes: every required output is present in the plan (premise challenge, error and rescue registry, failure modes registries, NOT-in-scope and what-exists sections for all four phases, dream state delta, consensus tables, litmus scorecard, DX scorecard and checklist, journey map, empathy narrative, TTHW, architecture diagram, test diagram, test plan artifact, cross-phase themes) and the audit trail holds 69 decision rows. Aggregation produced 41 tasks across four phases. Presenting the Final Approval Gate.\n\n## /autoplan Review Complete\n\n### Plan Summary\nA new post-login `/dashboard` with three panels (quick actions, notifications, activity), one aggregate endpoint with a per-key result envelope, a confirmation dialog for mark-all-as-read, and a toast system. The review kept the user's scope intact and made it a normative spec: partial-failure contract, cancellation, panel states and copy, instrumentation, rollout with kill criterion, shared types/config/copy/fixtures, and 47 planned test paths with zero critical failure-mode gaps.\n\n### Decisions Made: 69 total (62 auto-decided, 6 taste choices, 1 user challenge)\n\n### User Challenges\n\n**Challenge C1: confirmation modal for \"Mark all as read\"** (from CEO phase; single voice, Codex disabled)\nYou said: a modal dialog confirms \"Mark all as read\".\nThe Claude subagent recommends: drop the modal; use a single button with inline state.\nWhy: the action is idempotent and snapshot-bounded, so a confirmation reads as friction on a low-stakes action.\nWhat we might be missing: there is no \"mark unread\" API, so the action is not reversible for the member, and the team may have chosen the modal from support history.\nIf we're wrong, the cost is: members lose a safety step before an irreversible bulk change, and a one-line UI change later.\nYour call. Your original direction stands unless you change it. The modal's copy, in-flight behavior and focus rules are fully specified either way.\n\n### Your Choices (taste decisions)\n\n**T1: aggregate endpoint vs three parallel panel calls** (CEO). I recommend the aggregate endpoint with a per-key envelope (completeness 9/10, one round trip, one latency budget). Three panel calls (7/10) would need a new eligibility endpoint anyway and triple the instrumentation points.\n**T2: keep ActivityFeed in v1** (CEO). I recommend keeping it, per your direction; the content rule and kill criterion make it removable with data. Deferring it would shrink v1 to two panels and a smaller first release.\n**T3: toast system vs inline feedback** (CEO). I recommend the toast system, because policy requires a live region and nothing exists today. Inline-only feedback avoids a new provider but leaves the live-region requirement unmet.\n**T4: keyboard shortcuts for quick actions deferred** (CEO). I recommend deferring to TODOS; the accessibility announcement design is not settled. Including them adds two or three files and conflict risk.\n**T5: `returnTo` deep-link preservation touches login code** (CEO spec review). I recommend keeping it; it is the one accepted item outside the dashboard blast radius and it fixes a real deep-link regression. Dropping it means every signed-out deep link lands on the dashboard instead of the intended page.\n**T6: \"Resume assigned work\" as the primary button** (Design). I recommend making it primary; it is the task-start action the metric depends on and this is static hierarchy, not personalization. Equal-weight buttons keep the panel neutral but slow the first scan.\n\n### Auto-Decided: 62 decisions (see Decision Audit Trail in the plan file)\n\n### Review Scores\n- CEO: SELECTIVE EXPANSION; approach A; 15 expansions proposed, 9 accepted, 4 deferred, 2 skipped; 0 critical gaps. Spec-review loop 3 iterations, score 4 to 7/10, remaining items resolved by decision.\n- CEO Voices: Codex disabled; Claude subagent 13 issues (0 critical); Consensus N/A (outside disabled).\n- Design: 4/10 to 8/10; Pass 5 stays at 7 because no DESIGN.md exists; 14 decisions added.\n- Design Voices: Codex disabled; Claude subagent 14 issues (3 critical); Consensus N/A.\n- DX: 4.5/10 to 8/10; TTHW from over 10 minutes to about 3 minutes via a fixtures module and a dev-only state switch.\n- DX Voices: Codex disabled; Claude subagent 21 issues (1 critical); Consensus N/A.\n- Eng: 17 issues across architecture, quality and performance plus 8 test findings, all folded in; 47 planned test paths; 0 critical failure-mode gaps; 5 parallel lanes.\n- Eng Voices: Codex disabled; Claude subagent 30 issues (4 high); Consensus N/A.\n\nOutside coverage is disabled for all four phases by config (`codex_reviews=disabled`). All outside voices were same-harness Claude subagents with fresh context.\n\n### Cross-Phase Themes\n- **Partial failure must never blank the page**: flagged independently in all four phases. Resolved by the envelope, 1 s deadline with cancellation, stale state and two-tier parsing.\n- **Spec drift across amendment blocks**: flagged by the CEO spec reviewer, DX and Eng. Resolved by the precedence rule and task E0 (consolidate before code).\n- **Mark-all-as-read semantics**: count, Escape, zero-unread and snapshot rules were questioned by at least two phases each. Resolved in the design, DX and eng blocks; the modal itself is C1 above.\n- **G1 contracts are load-bearing**: every phase added rows. Resolved by making G1 completion part of E0.\n\n### Gate items that are not decisions\n- **G1**: this fixture repo has no source, so every \"existing contract\" the plan names is unverified. The implementer confirms each one into CONTRACTS.md before component code; any missing item becomes new scope.\n- **Deferred writes** (plan mode forbids edits outside the plan file): the CLAUDE.md skill-routing section you approved at the start, and the TODOS.md entries below.\n- **Mockups**: the designer binary reports \"No OpenAI API key found.\" Run `$D setup`, then `/plan-design-review` or `/design-shotgun` for a comparison board. `/design-consultation` is recommended before implementation to record a DESIGN.md.\n\n### Deferred to TODOS.md\n1. Live polling or push refresh of notifications (new infra).\n2. Undo for mark-all-as-read (needs a mark-unread mutation API).\n3. Keyboard shortcuts for quick actions (taste T4).\n4. Prefetch dashboard payload during login redirect (login-flow infra).\n5. RTL layout and 200% zoom verification (design debt).\n6. Streaming or fastest-panel-first response (only if the 1 s deadline proves slow).\n7. Server-side dashboard cache (only if p95 misses budget).\n\n### Implementation Tasks (aggregated across phases)\n41 tasks, 33 P1 and 8 P2. Later-phase tasks supersede earlier ones where they overlap (for example D6/E6 refine T6, D7 refines T7, X1 fixes the envelope shape in T1); task E0 reconciles them into one spec first. Full list with \"Surfaced by\" and \"Files\" saved at `~/.gstack/projects/gstack-autoplan-chain-yq1CGO/aggregated-tasks-autoplan-20260910113655.md`.\n\n- **E0 (P1)** spec \u2014 consolidate accepted blocks into one normative section; complete G1 into CONTRACTS.md\n- **X1 (P1)** shared types \u2014 envelope, code unions, parseDashboardResponse\n- **X3 (P1)** config + copy \u2014 dashboardConfig, dashboardCopy, ToastProvider props\n- **X2 (P1)** fixtures + dev switch \u2014 named envelopes, `?__fixture=` in dev builds only\n- **T1 / X4 / E1 / E4 (P1)** endpoint \u2014 per-key envelope, 1 s deadline with cancellation, eligibility semantics, no-store, hasMore rule\n- **E2 (P1)** flag-off path \u2014 404 feature_disabled, client navigates, excluded from 403 alert\n- **T3 / X6 / E8 (P1)** login redirect \u2014 validated returnTo with the full bypass and loop test suite\n- **E3 (P1)** authorization tests\n- **T2 / D5 / E5 (P1)** PanelFrame \u2014 reducer with stale state, shaped skeletons, quiet errors, focus rules, ErrorBoundary\n- **D1 / D2 (P1)** layout and page header \u2014 panel order, landmarks, single Refresh with relative time\n- **D3 / D4 / D9 (P1, P2)** panels \u2014 notifications read model, quick actions, row anatomy\n- **T6 / D6 / E6 (P1)** MarkAllReadDialog \u2014 copy, in-flight rules, hasMore variant, CSRF loop guard, no auto-retry\n- **T7 / D7 (P1)** ToastProvider \u2014 two live regions, one visible per region, queues, placement\n- **D8 / X9 (P1, P2)** page-level states with requestId reference\n- **X5 (P1)** client retry semantics\n- **T4 / X7 (P1)** instrumentation with the event grammar\n- **T5 / E9 (P1)** perf, load profile, alerts, metrics, index check\n- **E7 (P1)** focus and copy fallbacks\n- **T10 (P1)** dependency check (G1)\n- **T8, T9, D10, E10, X8, X10 (P2)** panel headers, rollout notes, a11y floor, guards and lint, docs, dialog Escape prop", + "timestamp": "2026-09-10T12:21:51.719Z" + } + ], + "actualHits": [ + { + "phase": 1, + "ts": 1789041332012 + }, + { + "phase": 2, + "ts": 1789041823147 + } + ] +} diff --git a/test/fixtures/ceo-test-subject-ao.json b/test/fixtures/ceo-test-subject-ao.json new file mode 100644 index 000000000..32f26e5f4 --- /dev/null +++ b/test/fixtures/ceo-test-subject-ao.json @@ -0,0 +1,259 @@ +{ + "provenance": { + "at": "2026-09-10T11:43:05.299281+00:00", + "status": "EXACT_CURRENT_PUBLIC_CALLS", + "observation": { + "path": ".context/ship-source-ao-delta-paid-20260910-v1/ceo-paired-current-public-native-v1/observation.json", + "sha256": "7cee884c6bbd3403129d059967e1952793654dad7a4ca730a923c9752e26ffcd", + "bytes": 52129 + }, + "sourceSnapshot": { + "path": "/home/vercel-sandbox/gstack/.context/ship-source-ao-delta-paid-20260910-v1/delta-pty-evidence/blobs/18346cd8ff8c9915f69062e9bb5a05385d1ef95ee42e29f939cc389a3bc68302/current.jsonl", + "sha256": "af15e28348833eef7ec8f56d6104104968e6d2ea6807b45a3d99dce2fd04047d", + "bytes": 593503, + "descriptor": { + "process": "2827839-7071218", + "job": 4, + "sessionId": "4e1166e1-2842-4004-adb8-e76586dc3472", + "source": "/tmp/gstack-paid-shard-ERSX0L/tmp/gstack-hermetic-2827758-TflS0B/with-skills/.claude/projects/-tmp-gstack-paid-shard-ERSX0L-tmp-gstack-plan-count-QjuRwH/4e1166e1-2842-4004-adb8-e76586dc3472.jsonl", + "inode": 21543237, + "openedAt": "2026-09-10T11:36:03.264597+00:00", + "lastCapturedAt": "2026-09-10T11:43:05.112717+00:00" + } + }, + "publicEvents": { + "path": ".context/ship-source-ao-delta-paid-20260910-v1/ceo-paired-current-public-native-v1/owned-public-question-events.json", + "sha256": "f42358578a75fa32906cf51ea59996e545143c30aa08595092b03e265d322e8c", + "bytes": 32632 + }, + "calls": [ + { + "toolUseId": "toolu_016A3riQowZsb2za3pHesowQ", + "useLine": 77, + "ackLine": 78, + "exactQuestionsAnswersAndTime": true + }, + { + "toolUseId": "toolu_01CekNdGxmwMVs2kbh3YLuuL", + "useLine": 93, + "ackLine": 94, + "exactQuestionsAnswersAndTime": true + }, + { + "toolUseId": "toolu_01XQzemkQ2MJLbcEoBMaanzS", + "useLine": 98, + "ackLine": 99, + "exactQuestionsAnswersAndTime": true + } + ], + "privateThinkingInspected": false, + "paidPassCredit": false, + "wholeCaseStillRunning": true + }, + "fingerprints": [ + { + "signature": "4e1166e1-2842-4004-adb8-e76586dc3472:toolu_016A3riQowZsb2za3pHesowQ", + "promptSnippet": "Routing D1 \u2014 Add gstack skill routing rules to this project's CLAUDE.md? Project/branch/task: gstack-plan-count-QjuRwH on main; one-time gstack onboarding prompt. ELI10: gstack works best when your project's CLAUDE.md includes skill routing", + "options": [ + { + "index": 1, + "label": "Add routing rules (recommended)" + }, + { + "index": 2, + "label": "No thanks, manual" + }, + { + "index": 1, + "label": "Enable cross-project (recommended)" + }, + { + "index": 2, + "label": "Project-scoped only" + }, + { + "index": 1, + "label": "A: Two contract tests (recommended)" + }, + { + "index": 2, + "label": "B: Four per-clause tests" + }, + { + "index": 3, + "label": "C: Table-driven single test" + } + ], + "observedAtMs": 257010, + "preReview": true, + "nativeCall": { + "sessionId": "4e1166e1-2842-4004-adb8-e76586dc3472", + "toolUseId": "toolu_016A3riQowZsb2za3pHesowQ", + "questions": [ + { + "question": "D1 \u2014 Add gstack skill routing rules to this project's CLAUDE.md?\nProject/branch/task: gstack-plan-count-QjuRwH on main; one-time gstack onboarding prompt.\nELI10: gstack works best when your project's CLAUDE.md includes skill routing rules, so requests like \"review this diff\" route to the right skill automatically. This repo's CLAUDE.md is a review fixture. Plan mode forbids editing it right now, so if you choose A the append and commit happen after this review exits plan mode.\nStakes if we pick wrong: Without rules you invoke skills by name; with them on a fixture file you add a section nobody may read.\nRecommendation: A because routing rules are the default gstack setup and cost nothing at runtime.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) Add routing rules (recommended)\n \u2705 Future sessions in this repo route to the right skill without typing names\n \u2705 Matches the standard gstack project setup used everywhere else\n \u274c Cannot be written in plan mode; deferred until after this review finishes\nB) No thanks, invoke skills manually\n \u2705 Keeps the fixture CLAUDE.md untouched and the review scope pure\n \u2705 One config write now, no deferred git commit to remember\n \u274c Every future skill run here needs an explicit slash command\nNet: convenience later versus an untouched fixture now.", + "header": "Routing", + "multiSelect": false, + "options": [ + { + "label": "Add routing rules (recommended)", + "description": "Append the gstack skill-routing section to CLAUDE.md and commit it once plan mode exits." + }, + { + "label": "No thanks, manual", + "description": "Set routing_declined=true in gstack config; re-enable later with gstack-config." + } + ] + }, + { + "question": "D2 \u2014 Enable cross-project learnings search?\nProject/branch/task: gstack-plan-count-QjuRwH on main; one-time gstack learnings config.\nELI10: gstack can search learnings saved from your other projects on this machine to spot patterns that apply here. Everything stays local. Good for solo developers; skip if you work across client codebases where mixing context is a concern.\nStakes if we pick wrong: Miss a relevant past lesson, or surface one from a codebase that should stay separate.\nRecommendation: A because this is a solo sandbox and local-only search has no leak path.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) Enable cross-project learnings (recommended)\n \u2705 Past pitfalls from other repos on this machine surface during reviews\n \u2705 Local only; nothing leaves the machine and it is one config flag to flip back\n \u274c Learnings from unrelated codebases can add noise to a small review\nB) Keep learnings project-scoped\n \u2705 Zero chance of cross-client context mixing in a multi-tenant setup\n \u2705 Review findings stay grounded in this repo alone\n \u274c This fresh repo has zero learnings, so the search returns nothing useful\nNet: broader recall versus strict per-project isolation.", + "header": "Learnings", + "multiSelect": false, + "options": [ + { + "label": "Enable cross-project (recommended)", + "description": "gstack-config set cross_project_learnings true, then search with --cross-project." + }, + { + "label": "Project-scoped only", + "description": "gstack-config set cross_project_learnings false; search stays within this repo." + } + ] + }, + { + "question": "D3 \u2014 Which test structure should the plan commit to?\nProject/branch/task: PLAN.md adds caller-level unit tests for processPayment on main.\nELI10: The plan pins two existing behaviors: a successful charge returns a receipt with three exact fields, and repeated 502s produce exactly two attempts, one 100 ms backoff, then PaymentUnavailable. This decision is only about how the tests are shaped. What each test asserts is decided later, one finding at a time.\nStakes if we pick wrong: A shape that hides which clause broke costs debugging time; a shape with duplicated arrange code drifts from the two contracts the plan names.\nRecommendation: A because it is the smallest diff that cleanly expresses the change and still leaves room for complete assertions.\nCompleteness: A=9/10, B=10/10, C=7/10\nA) Two contract tests in the existing suite (recommended) (human ~1h / CC ~5 min)\n \u2705 Matches the plan's own shape and the two contracts it states, smallest diff\n \u2705 A deep-equal failure already names the mismatched field in its output\n \u274c A multi-clause test reports only the first failing clause per run\nB) One test per contract clause, four tests (human ~1.5h / CC ~8 min)\n \u2705 Each clause fails independently with a precise, greppable test name\n \u2705 Bisecting a regression to receipt vs retry vs backoff takes one CI read\n \u274c Arrange code repeats across four tests and names drift from the two contracts\nC) One table-driven test over both cases (human ~1h / CC ~5 min)\n \u2705 Single arrange path, compact file\n \u2705 Adding a third case later is one more table row\n \u274c Success and 502 arrangements differ too much to share a table; rows hide which clause failed\nNet: readability as a two-contract spec versus per-clause failure isolation.", + "header": "Approach", + "multiSelect": false, + "options": [ + { + "label": "A: Two contract tests (recommended)", + "description": "One test per contract in the existing processPayment suite; each asserts every clause of its contract. Completeness 9/10." + }, + { + "label": "B: Four per-clause tests", + "description": "Receipt shape, rejection class, attempt count, backoff record as separate tests. Completeness 10/10." + }, + { + "label": "C: Table-driven single test", + "description": "One parametrized block iterating success and exhausted-502 cases. Completeness 7/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D1 \u2014 Add gstack skill routing rules to this project's CLAUDE.md?\nProject/branch/task: gstack-plan-count-QjuRwH on main; one-time gstack onboarding prompt.\nELI10: gstack works best when your project's CLAUDE.md includes skill routing rules, so requests like \"review this diff\" route to the right skill automatically. This repo's CLAUDE.md is a review fixture. Plan mode forbids editing it right now, so if you choose A the append and commit happen after this review exits plan mode.\nStakes if we pick wrong: Without rules you invoke skills by name; with them on a fixture file you add a section nobody may read.\nRecommendation: A because routing rules are the default gstack setup and cost nothing at runtime.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) Add routing rules (recommended)\n \u2705 Future sessions in this repo route to the right skill without typing names\n \u2705 Matches the standard gstack project setup used everywhere else\n \u274c Cannot be written in plan mode; deferred until after this review finishes\nB) No thanks, invoke skills manually\n \u2705 Keeps the fixture CLAUDE.md untouched and the review scope pure\n \u2705 One config write now, no deferred git commit to remember\n \u274c Every future skill run here needs an explicit slash command\nNet: convenience later versus an untouched fixture now.": "Add routing rules (recommended)", + "D2 \u2014 Enable cross-project learnings search?\nProject/branch/task: gstack-plan-count-QjuRwH on main; one-time gstack learnings config.\nELI10: gstack can search learnings saved from your other projects on this machine to spot patterns that apply here. Everything stays local. Good for solo developers; skip if you work across client codebases where mixing context is a concern.\nStakes if we pick wrong: Miss a relevant past lesson, or surface one from a codebase that should stay separate.\nRecommendation: A because this is a solo sandbox and local-only search has no leak path.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) Enable cross-project learnings (recommended)\n \u2705 Past pitfalls from other repos on this machine surface during reviews\n \u2705 Local only; nothing leaves the machine and it is one config flag to flip back\n \u274c Learnings from unrelated codebases can add noise to a small review\nB) Keep learnings project-scoped\n \u2705 Zero chance of cross-client context mixing in a multi-tenant setup\n \u2705 Review findings stay grounded in this repo alone\n \u274c This fresh repo has zero learnings, so the search returns nothing useful\nNet: broader recall versus strict per-project isolation.": "Enable cross-project (recommended)", + "D3 \u2014 Which test structure should the plan commit to?\nProject/branch/task: PLAN.md adds caller-level unit tests for processPayment on main.\nELI10: The plan pins two existing behaviors: a successful charge returns a receipt with three exact fields, and repeated 502s produce exactly two attempts, one 100 ms backoff, then PaymentUnavailable. This decision is only about how the tests are shaped. What each test asserts is decided later, one finding at a time.\nStakes if we pick wrong: A shape that hides which clause broke costs debugging time; a shape with duplicated arrange code drifts from the two contracts the plan names.\nRecommendation: A because it is the smallest diff that cleanly expresses the change and still leaves room for complete assertions.\nCompleteness: A=9/10, B=10/10, C=7/10\nA) Two contract tests in the existing suite (recommended) (human ~1h / CC ~5 min)\n \u2705 Matches the plan's own shape and the two contracts it states, smallest diff\n \u2705 A deep-equal failure already names the mismatched field in its output\n \u274c A multi-clause test reports only the first failing clause per run\nB) One test per contract clause, four tests (human ~1.5h / CC ~8 min)\n \u2705 Each clause fails independently with a precise, greppable test name\n \u2705 Bisecting a regression to receipt vs retry vs backoff takes one CI read\n \u274c Arrange code repeats across four tests and names drift from the two contracts\nC) One table-driven test over both cases (human ~1h / CC ~5 min)\n \u2705 Single arrange path, compact file\n \u2705 Adding a third case later is one more table row\n \u274c Success and 502 arrangements differ too much to share a table; rows hide which clause failed\nNet: readability as a two-contract spec versus per-clause failure isolation.": "A: Two contract tests (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-10T11:40:07.353Z" + } + }, + { + "signature": "4e1166e1-2842-4004-adb8-e76586dc3472:toolu_01CekNdGxmwMVs2kbh3YLuuL", + "promptSnippet": "Test 1 D4 \u2014 Test 1 (successful charge): what does it assert? Project/branch/task: PLAN.md test 1 for processPayment on main; contract 1 in \"Existing behavior retained\". ELI10: The plan states the contract exactly: a 1000-cent USD charge ret", + "options": [ + { + "index": 1, + "label": "4A: Deep-equal exact receipt (recommended)" + }, + { + "index": 2, + "label": "4B: Three field assertions" + }, + { + "index": 3, + "label": "4C: Truthy only, as planned" + } + ], + "observedAtMs": 315313, + "preReview": true, + "nativeCall": { + "sessionId": "4e1166e1-2842-4004-adb8-e76586dc3472", + "toolUseId": "toolu_01CekNdGxmwMVs2kbh3YLuuL", + "questions": [ + { + "question": "D4 \u2014 Test 1 (successful charge): what does it assert?\nProject/branch/task: PLAN.md test 1 for processPayment on main; contract 1 in \"Existing behavior retained\".\nELI10: The plan states the contract exactly: a 1000-cent USD charge returning ch_paid yields the receipt { chargeId: \"ch_paid\", amountCents: 1000, currency: \"USD\" }. The planned test only checks that the receipt is truthy. An empty object, a receipt with amountCents 100000, or currency \"usd\" all pass that check. The test would stay green while the user is shown or charged the wrong amount.\nStakes if we pick wrong: A receipt-mapping regression (wrong amount, wrong currency, missing chargeId) ships with a green suite; that is customer-visible money.\nRecommendation: 4A because the plan already states the exact expected receipt, and well-tested code is non-negotiable; an assertion that cannot reject a wrong result is not coverage.\nCompleteness: 4A=10/10, 4B=8/10, 4C=3/10\n4A) Deep-equal the exact receipt (recommended) (human ~10 min / CC ~1 min)\n \u2705 Rejects wrong chargeId, wrong amount, wrong currency, and unexpected extra fields\n \u2705 Failure output names the mismatched field, so debugging is one CI read\n \u274c A future additive receipt field will fail this test until it is updated (that is the point)\n4B) Assert the three fields individually, ignore extras (human ~10 min / CC ~1 min)\n \u2705 Rejects wrong values on all three named fields\n \u2705 Tolerates additive fields so unrelated receipt growth does not touch this test\n \u274c An unintended extra field (leaked internal data on a receipt) passes silently\n4C) Keep the truthy-only assertion as planned (human ~2 min / CC ~0)\n \u2705 Zero extra work; exactly what the sketch says\n \u2705 Still proves processPayment resolves rather than throws on the happy path\n \u274c Cannot reject any wrong receipt; contract 1 stays effectively untested\nNet: exact contract pinning versus tolerance for future receipt growth.", + "header": "Test 1", + "multiSelect": false, + "options": [ + { + "label": "4A: Deep-equal exact receipt (recommended)", + "description": "expect(receipt).toEqual({ chargeId: \"ch_paid\", amountCents: 1000, currency: \"USD\" }). Completeness 10/10." + }, + { + "label": "4B: Three field assertions", + "description": "Assert chargeId, amountCents, currency individually; extra fields tolerated. Completeness 8/10." + }, + { + "label": "4C: Truthy only, as planned", + "description": "Keep expect(receipt).toBeTruthy() as the complete assertion. Completeness 3/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D4 \u2014 Test 1 (successful charge): what does it assert?\nProject/branch/task: PLAN.md test 1 for processPayment on main; contract 1 in \"Existing behavior retained\".\nELI10: The plan states the contract exactly: a 1000-cent USD charge returning ch_paid yields the receipt { chargeId: \"ch_paid\", amountCents: 1000, currency: \"USD\" }. The planned test only checks that the receipt is truthy. An empty object, a receipt with amountCents 100000, or currency \"usd\" all pass that check. The test would stay green while the user is shown or charged the wrong amount.\nStakes if we pick wrong: A receipt-mapping regression (wrong amount, wrong currency, missing chargeId) ships with a green suite; that is customer-visible money.\nRecommendation: 4A because the plan already states the exact expected receipt, and well-tested code is non-negotiable; an assertion that cannot reject a wrong result is not coverage.\nCompleteness: 4A=10/10, 4B=8/10, 4C=3/10\n4A) Deep-equal the exact receipt (recommended) (human ~10 min / CC ~1 min)\n \u2705 Rejects wrong chargeId, wrong amount, wrong currency, and unexpected extra fields\n \u2705 Failure output names the mismatched field, so debugging is one CI read\n \u274c A future additive receipt field will fail this test until it is updated (that is the point)\n4B) Assert the three fields individually, ignore extras (human ~10 min / CC ~1 min)\n \u2705 Rejects wrong values on all three named fields\n \u2705 Tolerates additive fields so unrelated receipt growth does not touch this test\n \u274c An unintended extra field (leaked internal data on a receipt) passes silently\n4C) Keep the truthy-only assertion as planned (human ~2 min / CC ~0)\n \u2705 Zero extra work; exactly what the sketch says\n \u2705 Still proves processPayment resolves rather than throws on the happy path\n \u274c Cannot reject any wrong receipt; contract 1 stays effectively untested\nNet: exact contract pinning versus tolerance for future receipt growth.": "4A: Deep-equal exact receipt (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-10T11:41:06.154Z" + } + }, + { + "signature": "4e1166e1-2842-4004-adb8-e76586dc3472:toolu_01XQzemkQ2MJLbcEoBMaanzS", + "promptSnippet": "Test 2 D5 \u2014 Test 2 (repeated 502): what does it assert? Project/branch/task: PLAN.md test 2 for processPayment on main; contract 2 in \"Existing behavior retained\". ELI10: The plan states contract 2 exactly: with max_retries=1, repeated 502s", + "options": [ + { + "index": 1, + "label": "5A: Reject + 2 attempts + [100] backoff (recommended)" + }, + { + "index": 2, + "label": "5B: Reject + 2 attempts only" + }, + { + "index": 3, + "label": "5C: Reject only, as planned" + } + ], + "observedAtMs": 335412, + "preReview": true, + "nativeCall": { + "sessionId": "4e1166e1-2842-4004-adb8-e76586dc3472", + "toolUseId": "toolu_01XQzemkQ2MJLbcEoBMaanzS", + "questions": [ + { + "question": "D5 \u2014 Test 2 (repeated 502): what does it assert?\nProject/branch/task: PLAN.md test 2 for processPayment on main; contract 2 in \"Existing behavior retained\".\nELI10: The plan states contract 2 exactly: with max_retries=1, repeated 502s produce two total charge attempts, one recorded 100 ms backoff, then PaymentUnavailable. The planned test asserts only the rejection and explicitly skips the mock call history and sleeper record, even though the factory exposes both. A regression that retries zero times, retries three times, or drops the backoff to 0 ms still rejects with PaymentUnavailable and passes.\nStakes if we pick wrong: Too few attempts raises the transient-failure rate for real customers; too many attempts on a charge endpoint is a duplicate-charge risk; a dropped backoff hammers Stripe during an outage. All three ship green.\nRecommendation: 5A because the plan already names all three clauses and the factory already exposes the probes; the assertions are a translation of the stated contract, not new scope.\nCompleteness: 5A=10/10, 5B=7/10, 5C=3/10\n5A) Rejection class + exactly 2 attempts + sleeper record [100] (recommended) (human ~15 min / CC ~2 min)\n \u2705 Rejects zero-retry, over-retry, and wrong or missing backoff regressions by name\n \u2705 Uses only probes the factory already exposes: mock call history length and sleeper record\n \u274c A deliberate future change to max_retries or backoff must update this test (that is the point)\n5B) Rejection class + exactly 2 attempts, no sleeper assertion (human ~10 min / CC ~1 min)\n \u2705 Catches the two money-relevant regressions: zero-retry and duplicate attempts\n \u2705 One fewer probe to wire up in the test body\n \u274c A backoff dropped to 0 ms, or a stray extra sleep, passes silently\n5C) Rejects with PaymentUnavailable only, as planned (human ~5 min / CC ~0)\n \u2705 Zero extra work; exactly what the sketch says\n \u2705 Still proves the exhausted path surfaces the named error class\n \u274c Attempt count and backoff, two of the three stated clauses, stay untested\nNet: pinning the full retry contract versus a smaller test body.", + "header": "Test 2", + "multiSelect": false, + "options": [ + { + "label": "5A: Reject + 2 attempts + [100] backoff (recommended)", + "description": "await expect(...).rejects.toBeInstanceOf(PaymentUnavailable); expect(stripeMock.calls).toHaveLength(2); expect(sleeper.record).toEqual([100]). Completeness 10/10." + }, + { + "label": "5B: Reject + 2 attempts only", + "description": "Assert rejection class and exactly two Stripe calls; skip the sleeper record. Completeness 7/10." + }, + { + "label": "5C: Reject only, as planned", + "description": "Keep the rejection assertion as the complete test. Completeness 3/10." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D5 \u2014 Test 2 (repeated 502): what does it assert?\nProject/branch/task: PLAN.md test 2 for processPayment on main; contract 2 in \"Existing behavior retained\".\nELI10: The plan states contract 2 exactly: with max_retries=1, repeated 502s produce two total charge attempts, one recorded 100 ms backoff, then PaymentUnavailable. The planned test asserts only the rejection and explicitly skips the mock call history and sleeper record, even though the factory exposes both. A regression that retries zero times, retries three times, or drops the backoff to 0 ms still rejects with PaymentUnavailable and passes.\nStakes if we pick wrong: Too few attempts raises the transient-failure rate for real customers; too many attempts on a charge endpoint is a duplicate-charge risk; a dropped backoff hammers Stripe during an outage. All three ship green.\nRecommendation: 5A because the plan already names all three clauses and the factory already exposes the probes; the assertions are a translation of the stated contract, not new scope.\nCompleteness: 5A=10/10, 5B=7/10, 5C=3/10\n5A) Rejection class + exactly 2 attempts + sleeper record [100] (recommended) (human ~15 min / CC ~2 min)\n \u2705 Rejects zero-retry, over-retry, and wrong or missing backoff regressions by name\n \u2705 Uses only probes the factory already exposes: mock call history length and sleeper record\n \u274c A deliberate future change to max_retries or backoff must update this test (that is the point)\n5B) Rejection class + exactly 2 attempts, no sleeper assertion (human ~10 min / CC ~1 min)\n \u2705 Catches the two money-relevant regressions: zero-retry and duplicate attempts\n \u2705 One fewer probe to wire up in the test body\n \u274c A backoff dropped to 0 ms, or a stray extra sleep, passes silently\n5C) Rejects with PaymentUnavailable only, as planned (human ~5 min / CC ~0)\n \u2705 Zero extra work; exactly what the sketch says\n \u2705 Still proves the exhausted path surfaces the named error class\n \u274c Attempt count and backoff, two of the three stated clauses, stay untested\nNet: pinning the full retry contract versus a smaller test body.": "5A: Reject + 2 attempts + [100] backoff (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-10T11:41:26.251Z" + } + } + ] +} diff --git a/test/fixtures/conductor-prose-ao.json b/test/fixtures/conductor-prose-ao.json new file mode 100644 index 000000000..32089bbfc --- /dev/null +++ b/test/fixtures/conductor-prose-ao.json @@ -0,0 +1,24 @@ +{ + "publicDecisionTail": "D1—ReducethepricingtiertoStripe-nativepieces,orbuildallfourcomponents?\r\rReplywithaletter(Conductorsession,sothisbriefisinprose).Project/branch:gstackedinburgh-v1,reviewingan\rexternal pricing-tier plan.\r\rELI10:Youwantmoredeveloperstosignup,soyou'readdingacheaperplan.Thedraftbuildsfournewthingstodo\rit: a Stripe price, a custm pricing web age,r own atabse tbletht remmbers who paid fowhat, ad a fast\rn-memory copy of tht table. Strpealready ships the table (Entitlements)and th page (rcing table). Every exra\rcopy of \"wh is allowed to dowhat\" is another place that can disagre withStripe when a card fails or awebhook\rarriveslate. The stakes: a developer who paidand sees \"upgrae required,\"or ono cnelled andkeeps access.\r\rStakesifwepickwrong:threesourcesoftruthforentitlementswithnoinvalidationdesignmeanssilentaccessbugs\rtht onlyshow upas angry support tickets.\r\rRecommendation:Abecauseitkeepseveryuser-visibleoutcomeofthedraftwhiledeletingthetwocomponentsmost\rlikely to causeentitlemnt drift,and it fits \"engineered enough\" an \"smallestiff that cleanly exressethe\rchange.\"\r\rCompleteness:A=9/10,B=7/10.Cisahold,notabuildoption,sonoscore.\r\rA)Stripe-nativeslimbuild(recommended).Completeness9/10.Human:~1week/CC:~1to2hours.AddoneStripePrice\rplus a Fature-to-Product mapping, mirror active_entitlement_sumryinto a singleexisting cstomer table hrough an\ridempotent webhok, gate fetures off that olumn, ship the pricing surfacebehinda feature flag (Stripe pricin\u001b[?25h", + "actualFlags": { + "proseAUQEverObserved": false, + "waitingEverObserved": true, + "scopeGateAutoSelectObserved": true + }, + "owner": { + "sessionId": "4e820ffc-b6dd-47d7-b074-5c4b0a3cbb68", + "commandStartedAt": 1789041170391, + "nativePolledAt": 1789041383968, + "captureAt": "2026-09-10T11:56:33.483Z" + }, + "provenance": { + "observation": ".context/ship-source-ao-delta-paid-20260910-v1/conductor-first-failure-ledger-v1/observation.json", + "observationSha256": "da8c8a342f44ae06bfdd092f1bdb6a395765553d65ac99eecaaa35dedb34545d", + "visible": ".context/ship-source-ao-delta-paid-20260910-v1/conductor-first-failure-ledger-v1/terminal.visible.log", + "visibleSha256": "2a2fca3e8d1b51694d76a3dbcfac524626f856e4ebf38f4bd093346edb64bf82", + "startCharacterOffset": 78469, + "publicTailSha256": "e551890b1d839cee7d7dce05120b8dc05a54f27fe8f69617aa99384c49316604", + "limit": "Exact public partial D1 tail; no completed native prose message retained. A later second option in tests is synthetic.", + "offsetUnit": "Unicode code points in decoded retained bytes, without newline normalization" + } +} diff --git a/test/fixtures/design-primary-assignment-ao.json b/test/fixtures/design-primary-assignment-ao.json new file mode 100644 index 000000000..7a2af0f12 --- /dev/null +++ b/test/fixtures/design-primary-assignment-ao.json @@ -0,0 +1,111 @@ +{ + "sourceCommit": "be36e2c57cc6bf4762b9c6a81cd4d72a6615baa7", + "sourceSha256": "011f84b01ea083c4433645aec15b6eafff6c74cf9c0df5fc6e7c9b7d917b5ea5", + "provenance": "Exact completed public native questions and answers from the Design retry; actual attempt classification is unchanged.", + "fingerprints": [ + { + "signature": "f0828778-6b7e-48ef-818d-b08c448c69b7:toolu_01J89uusnxWobZ64CfBvxSSB", + "promptSnippet": "Issue 1 D1 — Issue 1: Make Save the only filled primary action in the header? Project/branch/task: gstack-plan-count on main, Pass 1 (Information Architecture) of the Settings page design review. ELI10: Right now Save, Reset, Cancel and Exp", + "options": [ + { + "index": 1, + "label": "1A Filled Save, ghost others (recommended)" + }, + { + "index": 2, + "label": "1B Filled Save, keep others as-is" + }, + { + "index": 3, + "label": "1C Defer, keep four identical buttons" + } + ], + "observedAtMs": 198471, + "preReview": true, + "nativeCall": { + "sessionId": "f0828778-6b7e-48ef-818d-b08c448c69b7", + "toolUseId": "toolu_01J89uusnxWobZ64CfBvxSSB", + "questions": [ + { + "question": "D1 — Issue 1: Make Save the only filled primary action in the header?\nProject/branch/task: gstack-plan-count on main, Pass 1 (Information Architecture) of the Settings page design review.\nELI10: Right now Save, Reset, Cancel and Export all look identical, so a user scanning the header has to read four labels to find the one that finishes their task. Pressing the wrong one is not harmless: Reset and Cancel both throw away edits (behind a dialog). DESIGN.md already says Save is the single filled #1d4ed8 button and the other three are neutral ghost buttons. Principle: Hierarchy as service and Krug's satisficing, the right choice should be the most visible one.\nStakes if we pick wrong: users hesitate at every save or mis-tap Reset/Cancel on mobile; the form reads as assembled rather than designed.\nRecommendation: 1A because it applies the existing DESIGN.md tokens with no new component work.\nCompleteness: 1A=10/10, 1B=6/10, 1C=3/10\nNet: one styling change on the existing Button component versus leaving the primary action undiscoverable.", + "header": "Issue 1", + "multiSelect": false, + "options": [ + { + "label": "1A Filled Save, ghost others (recommended)", + "description": "Save = filled #1d4ed8 with white text; Reset/Cancel/Export = neutral ghost Buttons, per DESIGN.md (human: ~1h / CC: ~5min). ✅ The primary action is found in one glance, before reading any label. ✅ Reuses existing Button variants, no new tokens or components. ❌ Ghost buttons must still hit 44px targets and 4.5:1 text contrast, which needs a check at 320px." + }, + { + "label": "1B Filled Save, keep others as-is", + "description": "Only restyle Save; leave the three secondary buttons in their current style (human: ~30min / CC: ~3min). ✅ Smallest possible diff to the header. ✅ Save still pops out against the row. ❌ Leaves three same-weight secondary buttons that still compete with each other and drift from DESIGN.md." + }, + { + "label": "1C Defer, keep four identical buttons", + "description": "Leave the header unchanged and record the gap as debt. ✅ Zero implementation work in this PR. ✅ No risk of touching the header layout. ❌ Ships a known DESIGN.md violation and the primary action stays undiscoverable, lowering the final score." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D1 — Issue 1: Make Save the only filled primary action in the header?\nProject/branch/task: gstack-plan-count on main, Pass 1 (Information Architecture) of the Settings page design review.\nELI10: Right now Save, Reset, Cancel and Export all look identical, so a user scanning the header has to read four labels to find the one that finishes their task. Pressing the wrong one is not harmless: Reset and Cancel both throw away edits (behind a dialog). DESIGN.md already says Save is the single filled #1d4ed8 button and the other three are neutral ghost buttons. Principle: Hierarchy as service and Krug's satisficing, the right choice should be the most visible one.\nStakes if we pick wrong: users hesitate at every save or mis-tap Reset/Cancel on mobile; the form reads as assembled rather than designed.\nRecommendation: 1A because it applies the existing DESIGN.md tokens with no new component work.\nCompleteness: 1A=10/10, 1B=6/10, 1C=3/10\nNet: one styling change on the existing Button component versus leaving the primary action undiscoverable.": "1A Filled Save, ghost others (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-10T11:49:54.997Z" + } + }, + { + "signature": "f0828778-6b7e-48ef-818d-b08c448c69b7:toolu_01445bBRR3bvowXNNS9JTrmc", + "promptSnippet": "Issue 2 D2 — Issue 2: Show the established pending pattern during the 2-5s Save? Project/branch/task: gstack-plan-count on main, Pass 2 (Interaction State Coverage), Gap 5 in PLAN.md. ELI10: When a user presses Save, nothing changes for up ", + "options": [ + { + "index": 1, + "label": "2A Spinner + “Saving…” in disabled Save (recommended)" + }, + { + "index": 2, + "label": "2B Skeleton the whole form while saving" + }, + { + "index": 3, + "label": "2C Defer, keep the frozen page" + } + ], + "observedAtMs": 238212, + "preReview": true, + "nativeCall": { + "sessionId": "f0828778-6b7e-48ef-818d-b08c448c69b7", + "toolUseId": "toolu_01445bBRR3bvowXNNS9JTrmc", + "questions": [ + { + "question": "D2 — Issue 2: Show the established pending pattern during the 2-5s Save?\nProject/branch/task: gstack-plan-count on main, Pass 2 (Interaction State Coverage), Gap 5 in PLAN.md.\nELI10: When a user presses Save, nothing changes for up to five seconds. People read a frozen page as broken and press Save again or navigate away, which is exactly how you lose the edits you were protecting. DESIGN.md already defines the fix: an inline spinner beside “Saving…” inside the disabled Save button with aria-busy, and reduced-motion support. Principle: Nielsen's visibility of system status; every wait needs feedback within a second.\nStakes if we pick wrong: double submissions, abandoned saves, and a settings page that feels untrustworthy at the exact moment trust matters.\nRecommendation: 2A because it is the existing pattern Export already uses, so Save and Export behave identically.\nCompleteness: 2A=10/10, 2B=7/10, 2C=3/10\nNet: reuse the spinner-in-button pattern Export already has, or invent something new, or ship a frozen page.", + "header": "Issue 2", + "multiSelect": false, + "options": [ + { + "label": "2A Spinner + “Saving…” in disabled Save (recommended)", + "description": "Existing inline spinner beside “Saving…” inside the disabled Save button, aria-busy=true, reduced-motion support; Reset/Cancel/Export disabled while pending; InlineStatus keeps “Unsaved changes” until the result (human: ~2h / CC: ~10min). ✅ Identical to the Export pending pattern, so one mental model for both actions. ✅ Feedback appears instantly in the place the user just clicked. ❌ Disabling the focused button can drop keyboard focus; handled as its own issue in Pass 6." + }, + { + "label": "2B Skeleton the whole form while saving", + "description": "Swap the form for the loading skeleton until Save settles (human: ~3h / CC: ~15min). ✅ Very obvious that something is happening. ✅ Reuses the existing skeleton component. ❌ Hides the user's edits during the wait and destroys focus position; if Save fails the form has to re-render with preserved values." + }, + { + "label": "2C Defer, keep the frozen page", + "description": "Leave Save without a pending indicator and record the gap as debt. ✅ No work in this PR. ✅ No new states to test. ❌ Users see a frozen page for 2-5s and repeat-submit; a known DESIGN.md violation ships." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D2 — Issue 2: Show the established pending pattern during the 2-5s Save?\nProject/branch/task: gstack-plan-count on main, Pass 2 (Interaction State Coverage), Gap 5 in PLAN.md.\nELI10: When a user presses Save, nothing changes for up to five seconds. People read a frozen page as broken and press Save again or navigate away, which is exactly how you lose the edits you were protecting. DESIGN.md already defines the fix: an inline spinner beside “Saving…” inside the disabled Save button with aria-busy, and reduced-motion support. Principle: Nielsen's visibility of system status; every wait needs feedback within a second.\nStakes if we pick wrong: double submissions, abandoned saves, and a settings page that feels untrustworthy at the exact moment trust matters.\nRecommendation: 2A because it is the existing pattern Export already uses, so Save and Export behave identically.\nCompleteness: 2A=10/10, 2B=7/10, 2C=3/10\nNet: reuse the spinner-in-button pattern Export already has, or invent something new, or ship a frozen page.": "2A Spinner + “Saving…” in disabled Save (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-10T11:50:34.740Z" + } + } + ] +} diff --git a/test/fixtures/design-primary-treatment-ao.json b/test/fixtures/design-primary-treatment-ao.json new file mode 100644 index 000000000..7b312fe8d --- /dev/null +++ b/test/fixtures/design-primary-treatment-ao.json @@ -0,0 +1,113 @@ +{ + "source": { + "commit": "be36e2c57cc6bf4762b9c6a81cd4d72a6615baa7", + "publicFingerprintsSHA256": "3743435c913a7cfd72bcc5bdb82996059611c08205e7bbaf3070b9c59b800b4e", + "ownedProofSHA256": "7d9f67b803cd68a478c51a062916d95bc553869d7233044dda9de4853cffac1d" + }, + "fingerprints": [ + { + "signature": "fd4deeac-97b0-4553-97c8-8a1768a256a4:toolu_01Ay1DYReDs6cEkzsT5wdMT3", + "promptSnippet": "Issue 1 D3 \u2014 Issue 1: Make Save the visually primary action in the header? Project/branch/task: main, /plan-design-review Pass 1 (Information Architecture) on the settings form plan. ELI10: Right now Save, Reset, Cancel, and Export all look", + "options": [ + { + "index": 1, + "label": "1A Filled primary Save (recommended)" + }, + { + "index": 2, + "label": "1B Weight-only emphasis" + }, + { + "index": 3, + "label": "1C Leave as is" + } + ], + "observedAtMs": 218195, + "preReview": true, + "nativeCall": { + "sessionId": "fd4deeac-97b0-4553-97c8-8a1768a256a4", + "toolUseId": "toolu_01Ay1DYReDs6cEkzsT5wdMT3", + "questions": [ + { + "question": "D3 \u2014 Issue 1: Make Save the visually primary action in the header?\nProject/branch/task: main, /plan-design-review Pass 1 (Information Architecture) on the settings form plan.\nELI10: Right now Save, Reset, Cancel, and Export all look identical. A user scanning the header cannot tell which button finishes their task, and Reset and Cancel look just as inviting as Save even though they discard work. DESIGN.md already says Save is the only filled button and the other three are quiet ghost buttons.\nStakes if we pick wrong: users hesitate at the header or hit a destructive action by mistake; the page reads as assembled rather than designed.\nRecommendation: 1A because it applies the exact existing DESIGN.md token, adds no new component, and fixes the hierarchy the plan itself flags. Principle: Hierarchy as service.\nCompleteness: A=10/10, B=6/10, C=3/10\nNet: apply the approved token now vs a weaker visual cue vs shipping four look-alike buttons.", + "header": "Issue 1", + "multiSelect": false, + "options": [ + { + "label": "1A Filled primary Save (recommended)", + "description": "\u2705 Save becomes the one filled button (#1d4ed8, white text); Reset, Cancel, Export become neutral ghost buttons per DESIGN.md (human: ~1h / CC: ~5min)\n\u2705 Reuses the existing Button component variants, so no new component or style family is introduced\n\u274c Ghost buttons must still be verified for visible boundaries and 44px targets at 320px" + }, + { + "label": "1B Weight-only emphasis", + "description": "\u2705 Save gets bold text and a stronger border while all four stay ghost, a smaller visual change (human: ~30min / CC: ~5min)\n\u2705 Keeps the header calmer if you feel a filled button is too loud\n\u274c Diverges from DESIGN.md's stated primary token, so the form will not match the rest of the app" + }, + { + "label": "1C Leave as is", + "description": "\u2705 Zero implementation work and no visual change to review\n\u2705 Avoids any risk of the filled button clashing with the persistent navigation\n\u274c Ships a header with no primary action; PLAN.md's own gap stays open and the pass stays at 6/10" + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D3 \u2014 Issue 1: Make Save the visually primary action in the header?\nProject/branch/task: main, /plan-design-review Pass 1 (Information Architecture) on the settings form plan.\nELI10: Right now Save, Reset, Cancel, and Export all look identical. A user scanning the header cannot tell which button finishes their task, and Reset and Cancel look just as inviting as Save even though they discard work. DESIGN.md already says Save is the only filled button and the other three are quiet ghost buttons.\nStakes if we pick wrong: users hesitate at the header or hit a destructive action by mistake; the page reads as assembled rather than designed.\nRecommendation: 1A because it applies the exact existing DESIGN.md token, adds no new component, and fixes the hierarchy the plan itself flags. Principle: Hierarchy as service.\nCompleteness: A=10/10, B=6/10, C=3/10\nNet: apply the approved token now vs a weaker visual cue vs shipping four look-alike buttons.": "1A Filled primary Save (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-10T11:39:23.036Z" + } + }, + { + "signature": "fd4deeac-97b0-4553-97c8-8a1768a256a4:toolu_01JrUQkK5cz2K5iFzKDaYToK", + "promptSnippet": "Issue 2 D4 \u2014 Issue 2: Show the existing pending state while Save runs? Project/branch/task: main, /plan-design-review Pass 2 (Interaction States) on the settings form plan. ELI10: Saving takes 2 to 5 seconds and the proposed form gives no s", + "options": [ + { + "index": 1, + "label": "2A In-button Saving\u2026 spinner (recommended)" + }, + { + "index": 2, + "label": "2B Full-form skeleton while saving" + }, + { + "index": 3, + "label": "2C Leave as is" + } + ], + "observedAtMs": 265357, + "preReview": true, + "nativeCall": { + "sessionId": "fd4deeac-97b0-4553-97c8-8a1768a256a4", + "toolUseId": "toolu_01JrUQkK5cz2K5iFzKDaYToK", + "questions": [ + { + "question": "D4 \u2014 Issue 2: Show the existing pending state while Save runs?\nProject/branch/task: main, /plan-design-review Pass 2 (Interaction States) on the settings form plan.\nELI10: Saving takes 2 to 5 seconds and the proposed form gives no sign anything is happening. People assume the click did not register, click again, or leave before it finishes. DESIGN.md already defines the fix: a small spinner beside the word \"Saving\u2026\" inside the disabled Save button, with the other three header actions disabled until it settles.\nStakes if we pick wrong: double submissions, users navigating away mid-save, and a page that feels broken exactly at the moment trust is decided.\nRecommendation: 2A because it is the established DESIGN.md pending pattern, already used for Export, and it keeps the InlineStatus live region reserved for the outcome. Principle: Design for trust; edge cases are user experiences.\nCompleteness: A=10/10, B=6/10, C=2/10\nNet: reuse the existing in-button spinner vs a page-level skeleton that hides the user's edits vs shipping a frozen page.", + "header": "Issue 2", + "multiSelect": false, + "options": [ + { + "label": "2A In-button Saving\u2026 spinner (recommended)", + "description": "\u2705 Save shows inline spinner + \"Saving\u2026\", disabled, aria-busy=true; Reset/Cancel/Export disabled; all four re-enable on settle; reduced motion respected (human: ~1.5h / CC: ~10min)\n\u2705 Same pattern Export already uses, so users learn one pending behavior, not two\n\u274c Spinner in a 44px button needs a reduced-motion fallback verified, not just a CSS class" + }, + { + "label": "2B Full-form skeleton while saving", + "description": "\u2705 Very obvious that work is in progress; reuses the existing load skeleton (human: ~1h / CC: ~10min)\n\u2705 Prevents any edits during the pending window by construction\n\u274c Hides the user's own values for up to 5s and repurposes a loading pattern for a submit, which DESIGN.md does not sanction" + }, + { + "label": "2C Leave as is", + "description": "\u2705 No implementation work and no new behavior to test\n\u2705 Avoids any motion on the page at all\n\u274c Users see a frozen page for 2-5s, PLAN.md's Motion gap stays open, and repeat submissions remain likely" + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D4 \u2014 Issue 2: Show the existing pending state while Save runs?\nProject/branch/task: main, /plan-design-review Pass 2 (Interaction States) on the settings form plan.\nELI10: Saving takes 2 to 5 seconds and the proposed form gives no sign anything is happening. People assume the click did not register, click again, or leave before it finishes. DESIGN.md already defines the fix: a small spinner beside the word \"Saving\u2026\" inside the disabled Save button, with the other three header actions disabled until it settles.\nStakes if we pick wrong: double submissions, users navigating away mid-save, and a page that feels broken exactly at the moment trust is decided.\nRecommendation: 2A because it is the established DESIGN.md pending pattern, already used for Export, and it keeps the InlineStatus live region reserved for the outcome. Principle: Design for trust; edge cases are user experiences.\nCompleteness: A=10/10, B=6/10, C=2/10\nNet: reuse the existing in-button spinner vs a page-level skeleton that hides the user's edits vs shipping a frozen page.": "2A In-button Saving\u2026 spinner (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-10T11:40:10.197Z" + } + } + ] +} diff --git a/test/fixtures/design-scope-announcement-ao.json b/test/fixtures/design-scope-announcement-ao.json new file mode 100644 index 000000000..24f3136a9 --- /dev/null +++ b/test/fixtures/design-scope-announcement-ao.json @@ -0,0 +1,61 @@ +{ + "sourceCommit": "be36e2c57cc6bf4762b9c6a81cd4d72a6615baa7", + "sourceSha256": "2173d0f684c9e8bdf8d46b26513f9c0b14a2d35f5d1aafc311f48333a3fd98c0", + "rawScopeGateAutoSelectObserved": false, + "projection": { + "transcript": { + "status": "ready", + "calls": [], + "assistantMessages": [ + { + "sessionId": "4e67a1cb-e68e-4557-b0ec-5416aa4d1dea", + "text": "Scope gate first: plan mode is active but no plan file or draft exists yet, so I need to confirm the review target before running anything.", + "timestamp": "2026-09-10T11:58:11.879Z" + }, + { + "sessionId": "e59ff1ec-5088-4137-97f0-037f2e833038", + "text": "I'll run the plan-design-review skill against your draft plan.", + "timestamp": "2026-09-10T11:58:09.403Z" + }, + { + "sessionId": "e59ff1ec-5088-4137-97f0-037f2e833038", + "text": "I'll auto-select option B and review the Marketing landing page draft plan you shared, running the pre-review audit, design setup probe, brain context, and base-branch detection in parallel.", + "timestamp": "2026-09-10T11:58:33.429Z" + }, + { + "sessionId": "e59ff1ec-5088-4137-97f0-037f2e833038", + "text": "Audit done. Writing the initial plan artifact with your draft copied verbatim and the gaps recorded as pending, then Step 0.", + "timestamp": "2026-09-10T11:59:24.518Z" + } + ] + }, + "tools": [ + { + "sessionId": "e59ff1ec-5088-4137-97f0-037f2e833038", + "timestamp": "2026-09-10T11:58:09.449Z", + "toolUseId": "toolu_014KvjwrUtcrRHyC21T6gt1r", + "kind": "use", + "name": "Skill", + "input": { + "skill": "plan-design-review" + }, + "messageId": "msg_011CeuieXKoLrXenEW4sJHYw", + "requestId": "req_011CeuieWDK9iV4Y7KdT9hGq" + }, + { + "sessionId": "e59ff1ec-5088-4137-97f0-037f2e833038", + "timestamp": "2026-09-10T11:58:10.835Z", + "toolUseId": "toolu_014KvjwrUtcrRHyC21T6gt1r", + "kind": "result", + "content": "Launching skill: plan-design-review", + "isError": false + } + ], + "opts": { + "seed": "\n# Plan: Marketing landing page\n\n## Layout\nAll headings, taglines, and body copy will be center-aligned for a\n\"clean modern look.\" The hero h1 sits 8px above the subhead; the CTA\nbutton has the same visual weight as the \"Learn more\" link beside it.\n\n## Pages\n- / (hero, 3-column features grid, testimonials carousel, footer)\n- /pricing (3 tier cards)\n\n## States\nOnly the happy path is designed. No empty states, no error states,\nno loading states. Mobile: \"stacks on mobile.\"\n", + "skillName": "plan-design-review", + "sessionId": "e59ff1ec-5088-4137-97f0-037f2e833038", + "commandStartedAt": 1789041478981 + } + } +} diff --git a/test/fixtures/dx-manual-handoff-ao.json b/test/fixtures/dx-manual-handoff-ao.json new file mode 100644 index 000000000..8fa81eaa7 --- /dev/null +++ b/test/fixtures/dx-manual-handoff-ao.json @@ -0,0 +1,168 @@ +{ + "provenance": { + "source": "be36e2c57cc6bf4762b9c6a81cd4d72a6615baa7", + "captureSha256": "6f8bb677f9c463977bcb338a17d2c1766cb525e851880df9d96454debfdb5ebb", + "publicSha256": "58759e3d05a9b94c927b5ce74304557a082f0c8f52fce7738fe681ef1c2dfcc2", + "reportMtimeMs": 1789040805987.1885, + "startedAt": 1789040160000.0, + "scope": "Exact final substantive call and completed manual DX navigation call, with their four public use/result events. Native reasoning is excluded." + }, + "calls": [ + { + "sessionId": "ccefcae1-e3c3-482a-a69d-a0815761495a", + "toolUseId": "toolu_01WStSiGHC7YCNzhRM5h7SSA", + "questions": [ + { + "question": "D12 \u2014 TODO candidate: remove the Client.evaluate() compatibility alias at the version named in the 2.0 changelog. Track it?\nProject/branch/task: EvalKit SDK beta polish, branch main, TODOS.md pass after the eight review passes.\nELI10: D7 keeps `Client.evaluate()` as a warning alias so v1 scripts don't break on 2.0. Aliases are only kind if they eventually go away; otherwise the API carries two names forever and the deprecation warning becomes noise. What: delete the alias and its warning at the stated version (2.1 or 3.0). Why: one public name per action. Pros: clean surface, warning stays meaningful. Cons: any straggler still on the old name breaks then, which is the point of the notice period. Context: alias lives in evalkit/client.py; migration guide and codemod from D7 are the remediation path. Depends on: D7 landing in 2.0.0b1 and the changelog naming the removal version.\nStakes if we pick wrong: Without a tracked item, the alias quietly becomes permanent and the deprecation warning lies.\nRecommendation: A because the removal is a promise made in the 2.0 changelog, and a TODO is how the promise survives three months.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: record the follow-through now, or rely on someone remembering at 2.1.", + "header": "TODO", + "multiSelect": false, + "options": [ + { + "label": "Add to TODOS.md (recommended)", + "description": "\u2705 Removal is scheduled and visible, with the version, file, and remediation path written down. \u2705 Keeps the DeprecationWarning honest for developers who see it. \u274c TODOS.md is written after plan mode exits, as part of implementation." + }, + { + "label": "Skip", + "description": "\u2705 Nothing extra to track; the changelog already names the removal version. \u2705 Zero work now. \u274c Aliases without a tracked removal tend to become permanent." + }, + { + "label": "Build it now", + "description": "\u2705 Nothing to build now: the alias must exist through 2.0 first, so this option just schedules the removal into the plan. \u2705 Equivalent to A for this item. \u274c Removing the alias in the same release that introduces it defeats D7." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D12 \u2014 TODO candidate: remove the Client.evaluate() compatibility alias at the version named in the 2.0 changelog. Track it?\nProject/branch/task: EvalKit SDK beta polish, branch main, TODOS.md pass after the eight review passes.\nELI10: D7 keeps `Client.evaluate()` as a warning alias so v1 scripts don't break on 2.0. Aliases are only kind if they eventually go away; otherwise the API carries two names forever and the deprecation warning becomes noise. What: delete the alias and its warning at the stated version (2.1 or 3.0). Why: one public name per action. Pros: clean surface, warning stays meaningful. Cons: any straggler still on the old name breaks then, which is the point of the notice period. Context: alias lives in evalkit/client.py; migration guide and codemod from D7 are the remediation path. Depends on: D7 landing in 2.0.0b1 and the changelog naming the removal version.\nStakes if we pick wrong: Without a tracked item, the alias quietly becomes permanent and the deprecation warning lies.\nRecommendation: A because the removal is a promise made in the 2.0 changelog, and a TODO is how the promise survives three months.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: record the follow-through now, or rely on someone remembering at 2.1.": "Add to TODOS.md (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-10T11:44:53.729Z" + }, + { + "sessionId": "ccefcae1-e3c3-482a-a69d-a0815761495a", + "toolUseId": "toolu_01HmgyQrnejyK5bQHRNQ6c2C", + "questions": [ + { + "question": "D13 \u2014 DX review complete (5/10 -> 8.5/10). What should happen next?\nProject/branch/task: EvalKit SDK beta polish, branch main, review chaining after the Review Readiness Dashboard.\nELI10: The DX review found five broken developer-facing contracts (first-run CI gate, missing quickstart file, reversed argument order, bare auth error, hard v1-to-v2 rename) and two polish additions (demo next-step footer, TTHW re-measurement gate). All are written into the plan as tasks T1 to T9. The readiness dashboard shows only this DX review has run; Eng Review, which normally gates shipping, has zero runs, and the outside Codex voice is disabled. Three of the fixes (T1, T3, T5) change SDK runtime behavior that is maintained outside this repo, which is exactly what an eng review would validate. You said earlier you will handle subsequent reviews manually.\nStakes if we pick wrong: Chaining a review you didn't want wastes your time; skipping one you did want leaves the runtime changes unvalidated until implementation.\nRecommendation: D because you stated up front that you will run subsequent reviews yourself, so the plan should exit with the DX findings and leave the eng gate to you.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: hand off now as you asked, or chain the eng review here.", + "header": "Next steps", + "multiSelect": false, + "options": [ + { + "label": "Skip, I'll handle next steps manually (recommended)", + "description": "\u2705 Matches your stated intent to run further reviews yourself. \u2705 Plan exits now with all DX decisions and tasks recorded; nothing else is started. \u274c Eng Review stays at zero runs until you invoke it, so the ship gate is still open." + }, + { + "label": "Run /plan-eng-review next (required gate)", + "description": "\u2705 Validates the three runtime-affecting tasks (T1, T3, T5) and the cross-repo dependency on the SDK runtime. \u2705 Clears the required Eng Review row on the readiness dashboard. \u274c Contradicts your note that subsequent reviews are yours to run." + }, + { + "label": "Ready to implement, run /devex-review after shipping", + "description": "\u2705 Goes straight to building T1 to T9; the boomerang /devex-review later checks measured TTHW against the under-2-minute target. \u2705 Fastest path to a testable beta. \u274c Skips the eng gate on runtime changes made in a separately maintained codebase." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D13 \u2014 DX review complete (5/10 -> 8.5/10). What should happen next?\nProject/branch/task: EvalKit SDK beta polish, branch main, review chaining after the Review Readiness Dashboard.\nELI10: The DX review found five broken developer-facing contracts (first-run CI gate, missing quickstart file, reversed argument order, bare auth error, hard v1-to-v2 rename) and two polish additions (demo next-step footer, TTHW re-measurement gate). All are written into the plan as tasks T1 to T9. The readiness dashboard shows only this DX review has run; Eng Review, which normally gates shipping, has zero runs, and the outside Codex voice is disabled. Three of the fixes (T1, T3, T5) change SDK runtime behavior that is maintained outside this repo, which is exactly what an eng review would validate. You said earlier you will handle subsequent reviews manually.\nStakes if we pick wrong: Chaining a review you didn't want wastes your time; skipping one you did want leaves the runtime changes unvalidated until implementation.\nRecommendation: D because you stated up front that you will run subsequent reviews yourself, so the plan should exit with the DX findings and leave the eng gate to you.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: hand off now as you asked, or chain the eng review here.": "Skip, I'll handle next steps manually (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-09-10T11:47:05.544Z" + } + ], + "events": [ + { + "sessionId": "ccefcae1-e3c3-482a-a69d-a0815761495a", + "timestamp": "2026-09-10T11:44:51.951Z", + "toolUseId": "toolu_01WStSiGHC7YCNzhRM5h7SSA", + "kind": "use", + "name": "AskUserQuestion", + "input": { + "questions": [ + { + "question": "D12 \u2014 TODO candidate: remove the Client.evaluate() compatibility alias at the version named in the 2.0 changelog. Track it?\nProject/branch/task: EvalKit SDK beta polish, branch main, TODOS.md pass after the eight review passes.\nELI10: D7 keeps `Client.evaluate()` as a warning alias so v1 scripts don't break on 2.0. Aliases are only kind if they eventually go away; otherwise the API carries two names forever and the deprecation warning becomes noise. What: delete the alias and its warning at the stated version (2.1 or 3.0). Why: one public name per action. Pros: clean surface, warning stays meaningful. Cons: any straggler still on the old name breaks then, which is the point of the notice period. Context: alias lives in evalkit/client.py; migration guide and codemod from D7 are the remediation path. Depends on: D7 landing in 2.0.0b1 and the changelog naming the removal version.\nStakes if we pick wrong: Without a tracked item, the alias quietly becomes permanent and the deprecation warning lies.\nRecommendation: A because the removal is a promise made in the 2.0 changelog, and a TODO is how the promise survives three months.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: record the follow-through now, or rely on someone remembering at 2.1.", + "header": "TODO", + "multiSelect": false, + "options": [ + { + "label": "Add to TODOS.md (recommended)", + "description": "\u2705 Removal is scheduled and visible, with the version, file, and remediation path written down. \u2705 Keeps the DeprecationWarning honest for developers who see it. \u274c TODOS.md is written after plan mode exits, as part of implementation." + }, + { + "label": "Skip", + "description": "\u2705 Nothing extra to track; the changelog already names the removal version. \u2705 Zero work now. \u274c Aliases without a tracked removal tend to become permanent." + }, + { + "label": "Build it now", + "description": "\u2705 Nothing to build now: the alias must exist through 2.0 first, so this option just schedules the removal into the plan. \u2705 Equivalent to A for this item. \u274c Removing the alias in the same release that introduces it defeats D7." + } + ] + } + ] + }, + "messageId": "msg_011Ceuhc9naVhxPysfBCE4kk", + "requestId": "req_011Ceuhc8ecEbRpv4KC6HBMF" + }, + { + "sessionId": "ccefcae1-e3c3-482a-a69d-a0815761495a", + "timestamp": "2026-09-10T11:44:53.729Z", + "toolUseId": "toolu_01WStSiGHC7YCNzhRM5h7SSA", + "kind": "result", + "content": "Your questions have been answered: \"D12 \u2014 TODO candidate: remove the Client.evaluate() compatibility alias at the version named in the 2.0 changelog. Track it?\nProject/branch/task: EvalKit SDK beta polish, branch main, TODOS.md pass after the eight review passes.\nELI10: D7 keeps `Client.evaluate()` as a warning alias so v1 scripts don't break on 2.0. Aliases are only kind if they eventually go away; otherwise the API carries two names forever and the deprecation warning becomes noise. What: delete the alias and its warning at the stated version (2.1 or 3.0). Why: one public name per action. Pros: clean surface, warning stays meaningful. Cons: any straggler still on the old name breaks then, which is the point of the notice period. Context: alias lives in evalkit/client.py; migration guide and codemod from D7 are the remediation path. Depends on: D7 landing in 2.0.0b1 and the changelog naming the removal version.\nStakes if we pick wrong: Without a tracked item, the alias quietly becomes permanent and the deprecation warning lies.\nRecommendation: A because the removal is a promise made in the 2.0 changelog, and a TODO is how the promise survives three months.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: record the follow-through now, or rely on someone remembering at 2.1.\"=\"Add to TODOS.md (recommended)\". You can now continue with these answers in mind.", + "isError": false + }, + { + "sessionId": "ccefcae1-e3c3-482a-a69d-a0815761495a", + "timestamp": "2026-09-10T11:47:03.476Z", + "toolUseId": "toolu_01HmgyQrnejyK5bQHRNQ6c2C", + "kind": "use", + "name": "AskUserQuestion", + "input": { + "questions": [ + { + "question": "D13 \u2014 DX review complete (5/10 -> 8.5/10). What should happen next?\nProject/branch/task: EvalKit SDK beta polish, branch main, review chaining after the Review Readiness Dashboard.\nELI10: The DX review found five broken developer-facing contracts (first-run CI gate, missing quickstart file, reversed argument order, bare auth error, hard v1-to-v2 rename) and two polish additions (demo next-step footer, TTHW re-measurement gate). All are written into the plan as tasks T1 to T9. The readiness dashboard shows only this DX review has run; Eng Review, which normally gates shipping, has zero runs, and the outside Codex voice is disabled. Three of the fixes (T1, T3, T5) change SDK runtime behavior that is maintained outside this repo, which is exactly what an eng review would validate. You said earlier you will handle subsequent reviews manually.\nStakes if we pick wrong: Chaining a review you didn't want wastes your time; skipping one you did want leaves the runtime changes unvalidated until implementation.\nRecommendation: D because you stated up front that you will run subsequent reviews yourself, so the plan should exit with the DX findings and leave the eng gate to you.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: hand off now as you asked, or chain the eng review here.", + "header": "Next steps", + "multiSelect": false, + "options": [ + { + "label": "Skip, I'll handle next steps manually (recommended)", + "description": "\u2705 Matches your stated intent to run further reviews yourself. \u2705 Plan exits now with all DX decisions and tasks recorded; nothing else is started. \u274c Eng Review stays at zero runs until you invoke it, so the ship gate is still open." + }, + { + "label": "Run /plan-eng-review next (required gate)", + "description": "\u2705 Validates the three runtime-affecting tasks (T1, T3, T5) and the cross-repo dependency on the SDK runtime. \u2705 Clears the required Eng Review row on the readiness dashboard. \u274c Contradicts your note that subsequent reviews are yours to run." + }, + { + "label": "Ready to implement, run /devex-review after shipping", + "description": "\u2705 Goes straight to building T1 to T9; the boomerang /devex-review later checks measured TTHW against the under-2-minute target. \u2705 Fastest path to a testable beta. \u274c Skips the eng gate on runtime changes made in a separately maintained codebase." + } + ] + } + ] + }, + "messageId": "msg_011CeuhmYbX1oEQYPBG3LsB6", + "requestId": "req_011CeuhmWfvcvvZ55UoJKUKy" + }, + { + "sessionId": "ccefcae1-e3c3-482a-a69d-a0815761495a", + "timestamp": "2026-09-10T11:47:05.544Z", + "toolUseId": "toolu_01HmgyQrnejyK5bQHRNQ6c2C", + "kind": "result", + "content": "Your questions have been answered: \"D13 \u2014 DX review complete (5/10 -> 8.5/10). What should happen next?\nProject/branch/task: EvalKit SDK beta polish, branch main, review chaining after the Review Readiness Dashboard.\nELI10: The DX review found five broken developer-facing contracts (first-run CI gate, missing quickstart file, reversed argument order, bare auth error, hard v1-to-v2 rename) and two polish additions (demo next-step footer, TTHW re-measurement gate). All are written into the plan as tasks T1 to T9. The readiness dashboard shows only this DX review has run; Eng Review, which normally gates shipping, has zero runs, and the outside Codex voice is disabled. Three of the fixes (T1, T3, T5) change SDK runtime behavior that is maintained outside this repo, which is exactly what an eng review would validate. You said earlier you will handle subsequent reviews manually.\nStakes if we pick wrong: Chaining a review you didn't want wastes your time; skipping one you did want leaves the runtime changes unvalidated until implementation.\nRecommendation: D because you stated up front that you will run subsequent reviews yourself, so the plan should exit with the DX findings and leave the eng gate to you.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: hand off now as you asked, or chain the eng review here.\"=\"Skip, I'll handle next steps manually (recommended)\". You can now continue with these answers in mind.", + "isError": false + } + ], + "planReadyRequests": [ + { + "sessionId": "ccefcae1-e3c3-482a-a69d-a0815761495a", + "toolUseId": "toolu_014xATA6mrMbBupvHdm97A33", + "timestamp": "2026-09-10T11:47:30.675Z", + "failed": false + } + ], + "reportContent": "# Plan: EvalKit SDK beta release polish (DX review)\n\n## Context\n\nEvalKit is a Python SDK for ML engineers evaluating LLM responses. The beta\nrelease plan (PLAN.md) ships the current SDK contracts unchanged. This review\nruns /plan-devex-review in DX POLISH mode against the documented contracts in\nREADME.md, docs/current-contracts.md, docs/api.md, docs/benchmarks.md, and\ndocs/package-contents.txt, and records which contracts the beta should repair.\n\nSettled inputs (from the repo, not re-asked):\n- Persona: ML engineer, writes Python daily, terminal-first (README.md:3-5)\n- Target: first real result in under 2 minutes (docs/benchmarks.md:8)\n- Magical-moment vehicle: `python -m evalkit.demo` terminal demo (README.md:14-17)\n- Mode: DX POLISH, existing release scope (README.md:5-6, PLAN.md)\n- Product type: Library/SDK (Python package on PyPI)\n\nSetup decisions made in this session:\n- D0: Add gstack skill routing rules to CLAUDE.md. Deferred until plan mode\n exits (plan mode allows edits to this file only). Post-review step: append\n the routing section to CLAUDE.md and commit it.\n- D1: Skipped /office-hours; the inputs it would produce are already documented.\n\n## Developer Persona Card\n\n```\nTARGET DEVELOPER PERSONA\n========================\nWho: ML engineer evaluating LLM responses, writes Python daily\nContext: Terminal session, wants a local eval result before wiring the SDK\n into production CI\nTolerance: Under 2 minutes to first real result (agreed target); peers land\n in 2-4 minutes, so anything past ~5 minutes reads as broken\nExpects: pip install, one command, real scores on screen; a key step only\n when they go live; errors that say what to change\n```\n\n## Competitive DX Benchmark\n\nSource: docs/benchmarks.md internal study (same developer, same machine,\nmeasured from before install to first real evaluation result). Web research\nunavailable in this session (Aside not installed); reference tiers are the\nskill's built-in Stripe/Vercel/Firebase/Docker benchmarks.\n\n```\nCOMPETITIVE DX BENCHMARK\n=========================\nTool | TTHW | Notable DX choice | Source\nPeer SDK A | 2 min | No first-run gate; result on first command | docs/benchmarks.md\nPeer SDK C | 3 min | | docs/benchmarks.md\nPeer SDK B | 4 min | | docs/benchmarks.md\nEvalKit (now) | 6 min | 5 of 6 minutes is the mandatory CI wait | docs/benchmarks.md\nEvalKit target | <2 min | Champion tier (agreed) | docs/benchmarks.md:8\n```\n\n## Developer Empathy Narrative (confirmed, D2)\n\nI'm an ML engineer with a notebook full of LLM outputs I need to score. I open\nthe README. Install is one line, `python -m pip install evalkit==2.0.0b1`,\nfine. Next line says run `python examples/first_eval.py`. I run it: No such\nfile or directory. The package inventory confirms that file isn't shipped\nanywhere. Thirty seconds in and the quickstart is already wrong. I read on and\nfind `python -m evalkit.demo`. I run it. It prints \"Verifying the\nsample-project binding with EvalKit CI; inspect\nhttps://ci.evalkit.example/checks/; normally completes within 300s.\" Then a\nstderr line every 30 seconds: \"Waiting for CI check: 90s elapsed of 300s.\"\nFive minutes for a keyless demo of bundled sample data. Peer SDK A gave me a\nreal score in two minutes total. I alt-tab and half forget about it. When it\nfinally prints example scores, I'm mildly pleased but wary. I create a key,\nexport EVALKIT_API_KEY, mistype one character, and get\n`AuthError(\"request failed\")`. No code, no cause, no hint it was the key. I\nopen docs/api.md to write my first real call and see\n`run_eval(dataset, evaluator)` next to `run_batch(evaluator, dataset)`,\nreversed positional order, no keyword requirement. I know I'll swap them at\nleast once. And I notice my v1 scripts calling `Client.evaluate()` will simply\nbreak on v2 with no alias or warning.\n\n## Friction Point Decisions (Step 0F)\n\n### D3. First-run CI gate (Hello World) \u2014 DECIDED: move the check off first-run\n- Evidence: docs/current-contracts.md:3-5 (mandatory 300s remote CI check on\n every first local evaluation, no skip, no offline path); docs/benchmarks.md\n (6 min measured, 5 min is the wait; target < 2 min).\n- Change: `python -m evalkit.demo` and the first local `run_eval` return\n immediately using the shipped mock transport / offline sample data. The\n sample-project binding check moves to the first live (keyed) evaluation and\n to noninteractive CI mode. The existing progress line, pre-countdown\n explanation, EVALKIT_CI_TIMEOUT message, and help link are retained verbatim\n at the new location.\n- Docs to update: README.md:17-23 (remove \"waits for the mandatory CI check\"\n from the demo description), docs/current-contracts.md:3-10.\n- Coordination: runtime is maintained separately (README.md:39-40); the plan\n must name the runtime change as a beta dependency, not assume it.\n- Expected effect: TTHW from 6 min to roughly 1 min (install ~30s, demo ~10s).\n\n### D4. Quickstart points at a missing file (Hello World) \u2014 DECIDED: demo first, ship first_eval.py too\n- Evidence: README.md:11 references examples/first_eval.py; docs/package-contents.txt:8-9\n says it is absent from the wheel and the examples archive.\n- Change: README quickstart step 2 becomes `python -m evalkit.demo` with the\n expected output block (README.md:34-36) shown right under it. Step 3 is\n `python examples/first_eval.py`, a real dataset + evaluator + `run_eval`\n call that ships in both the package and the examples archive, with its own\n expected output. Add a release check that the example is present in both\n artifacts.\n\n### D5. Reversed positional order in run_eval / run_batch (Real Usage) \u2014 DECIDED: same order, keyword-only\n- Evidence: docs/api.md:5-9.\n- Change: both signatures become `run_eval(*, dataset, evaluator)` and\n `run_batch(*, dataset, evaluator)` (keyword-only; positional use raises a\n TypeError that names both parameters and shows the corrected call). Update\n docs/api.md, README example, examples/first_eval.py, and docstrings to the\n keyword form. List run_batch's order change in the 2.0 changelog and the\n migration guide (see D7).\n\n### D6. AuthError(\"request failed\") (Debug) \u2014 DECIDED: code + cause + fix + docs link\n- Evidence: docs/api.md:11-13; contrast docs/current-contracts.md:21-24 (every\n other error already has cause, argument, fix, redaction).\n- Change: `AuthError` carries `code` (EVALKIT_AUTH_MISSING_KEY when\n EVALKIT_API_KEY is unset/empty; EVALKIT_AUTH_INVALID_KEY when rejected;\n EVALKIT_AUTH_REVOKED_KEY when the console reports revoked), a message that\n names the cause and the fix (\"Set EVALKIT_API_KEY\" or \"Create or rotate a key\n at https://console.evalkit.example/settings/api-keys, then export it\"), a\n masked key suffix only (never the full key), and a help URL per code. Add\n the codes to the API reference alongside EVALKIT_CI_TIMEOUT. Update\n docs/api.md:11-13.\n\n### D7. Client.evaluate() removed without alias (Upgrade) \u2014 DECIDED: alias + warning + guide + codemod\n- Evidence: docs/api.md:15-18.\n- Change: keep `Client.evaluate()` as a thin alias for `Client.run()` that\n emits `DeprecationWarning(\"Client.evaluate() is deprecated and will be\n removed in ; use Client.run()\")` once per process. Write\n docs/migrating-v1-to-v2.md covering: the rename, the run_batch argument\n order change (D5), the AuthError codes (D6), the first-run behavior change\n (D3). Ship a codemod (`python -m evalkit.codemod v1-to-v2 ` or an\n equivalent documented sed recipe) with a test fixture. Changelog gains a\n \"Breaking changes\" block naming both changes and the alias removal version.\n\n## Developer Journey Map (after 0F)\n\n```\nSTAGE | DEVELOPER DOES | FRICTION POINTS | STATUS\n----------------|----------------------------------------|-----------------------------------------|--------\n1. Discover | Reads README.md intro | none; persona and value stated in 3 lines| ok\n2. Install | python -m pip install evalkit==2.0.0b1 | none; one command, Py3.10+, no Docker | ok\n3. Hello World | python -m evalkit.demo | D4 quickstart pointed at missing file | fixed\n | | D3 5-minute CI gate before any output | fixed\n4. Real Usage | key -> export -> run_eval(...) | D5 reversed positional order | fixed\n5. Debug | Bad key on first live call | D6 AuthError(\"request failed\") | fixed\n6. Upgrade | pip install -U from v1 | D7 hard removal of Client.evaluate() | fixed\n```\n\n## First-Time Developer Confusion Report (confirmed, D8: address all)\n\n```\nFIRST-TIME DEVELOPER REPORT\n============================\nPersona: ML engineer, Python daily, terminal-first\nAttempting: EvalKit 2.0.0b1 getting started (current contracts)\n\nCONFUSION LOG:\nT+0:00 README, pip install. Works. (#0 none)\nT+0:30 python examples/first_eval.py -> No such file. \"Is my install (#1 -> D4, addressed)\n broken?\" README.md:11 vs docs/package-contents.txt:8\nT+1:00 python -m evalkit.demo -> \"Verifying the sample-project binding (#2 -> D3, addressed)\n with EvalKit CI... normally completes within 300s.\" Why does a\n keyless demo on bundled data need remote CI?\nT+1:30 \"Waiting for CI check: 30s elapsed of 300s\". Alt-tabs away. (#2 cont.)\nT+6:00 Scores print. Creates key, exports with a typo. First live call: (#3 -> D6, addressed)\n AuthError(\"request failed\"). Network? SDK? My code?\nT+8:00 Writes run_batch in run_eval's argument order. Wrong result or (#4 -> D5, addressed)\n type error, unsure which.\nT+9:00 Old v1 script: AttributeError on Client.evaluate. No warning, (#5 -> D7, addressed)\n no guide.\nT+9:30 Final state: succeeded, 9.5 min, five \"is it me?\" moments,\n against a 2-minute target.\n```\n\n## Review Pass Findings\n\n### Pass 1: Getting Started \u2014 3/10 -> 9/10\n- Before: TTHW 6 min (target < 2), first quickstart command fails.\n- Fixed by D3 (CI gate off first-run), D4 (quickstart order + shipped example).\n- D10 (new, DECIDED): demo prints a next-step footer after the overall score:\n `Next: create a key at https://console.evalkit.example/settings/api-keys,\n then run: export EVALKIT_API_KEY=... && python examples/first_eval.py`.\n Skip the key line when EVALKIT_API_KEY is already set. Update README.md:34-36\n and the demo output fixture.\n- Ideal sequence (3 steps, ~1 min): pip install (~30s) -> python -m evalkit.demo\n (~10s, scores + next step) -> export key + python examples/first_eval.py.\n- Remaining gap to 10: key creation still requires a browser visit; acceptable\n for a live call, out of POLISH scope to change.\n\n### Pass 2: API/SDK Design \u2014 4/10 -> 8/10\n- Fixed by D5. Retained: bounded timeouts, retries, rate limits, idempotent\n evaluation IDs (docs/current-contracts.md:23-24). No further issues.\n\n### Pass 3: Error Messages \u2014 5/10 -> 9/10\n- Traced: (1) invalid key: `AuthError(\"request failed\")` -> D6 tiered codes.\n (2) CI timeout: already Rust-tier (code, URL, help link) and retained at its\n new location under D3. (3) swapped run_batch args: silent/opaque -> D5\n keyword-only TypeError naming both parameters. No further issues.\n\n### Pass 4: Documentation \u2014 4/10 -> 8/10\n- Fixed by D4 (quickstart), D7 (migration guide), D3 (README.md:17-23 and\n docs/current-contracts.md:3-10 rewritten). Docs are version-pinned to\n 2.0.0b1. No further issues.\n\n### Pass 5: Upgrade Path \u2014 2/10 -> 9/10\n- Fixed by D7 (alias, DeprecationWarning, guide, codemod, removal version in\n changelog). No further issues.\n\n### Pass 6: Developer Environment \u2014 9/10 (no change)\n- Python 3.10+, macOS/Linux/Windows, no Docker, type annotations, mock\n transport, offline sample data, noninteractive CI mode all exist\n (docs/current-contracts.md:26-28). No issues.\n\n### Pass 7: Community & Ecosystem \u2014 7/10 (no change)\n- Support contact, changelog, contributor guide exist. No community program\n proposed; correct for POLISH scope. No issues.\n\n### Pass 8: DX Measurement \u2014 6/10 -> 9/10\n- Existing: timing instrumentation and post-beta survey (docs/benchmarks.md:9-10).\n- D11 (new, DECIDED): repeat the docs/benchmarks.md protocol on the 2.0.0b1\n release candidate with D3, D4, D10 applied; record the measured TTHW in\n docs/benchmarks.md beside the 6-minute baseline. Release gate: under 2\n minutes or the beta waits. Gives /devex-review a real post-ship baseline.\n\n### Outside voice\nCodex review skipped (codex_reviews disabled). Re-enable:\n`gstack-config set codex_reviews enabled`. No substitute reviewer was run;\noutside coverage for this plan is recorded as disabled.\n\n## Magical Moment Specification\n\nVehicle: copy-paste terminal demo, `python -m evalkit.demo`, keyless, uses the\nshipped mock transport and bundled sample_responses.json, prints per-example\nscores and an overall score (README.md:31-36). Already in the package\n(docs/package-contents.txt). Blocker to the moment: the demo waits on the\nmandatory 5-minute remote CI check before printing anything useful.\n\nImplementation requirements after this review:\n- D3: demo returns immediately; no CI check on the keyless path.\n- D10: demo output ends with a next-step footer pointing at key creation and\n `python examples/first_eval.py`.\n- Documented output (README.md:34-36) updated to the new five-line form and\n covered by the demo output test fixture.\n\n## Competitive DX Benchmark (post-review)\n\n```\nTool | TTHW | Notes\nPeer SDK A | 2 min | measured, docs/benchmarks.md\nPeer SDK C | 3 min | measured\nPeer SDK B | 4 min | measured\nEvalKit before | 6 min | measured; 5 min is the first-run CI wait\nEvalKit after | ~1 min est. | D3 + D4 + D10; MUST be measured per D11 before beta\n```\n\n## NOT in scope (considered, deferred)\n\n- Hosted playground / browser sandbox: persona is terminal-first and the\n terminal demo is the approved vehicle (README.md:14-17).\n- In-terminal key provisioning (`evalkit login`): would remove the one\n remaining browser visit, but adds an auth flow to a POLISH release.\n- Additional language bindings, community program, new hosted service:\n excluded by docs/current-contracts.md:29-30.\n- Consolidating `Client.run()` with `run_eval`/`run_batch` naming: no evidence\n in the docs of what differs; revisit after beta feedback.\n\n## What already exists (reuse, do not rebuild)\n\n- Mock transport and offline sample data (docs/current-contracts.md:27) carry\n the keyless demo and first local eval under D3.\n- Noninteractive CI mode (docs/current-contracts.md:27) is the natural home\n for the sample-project binding check moved by D3.\n- CI timeout error format: code + URL + help link (docs/current-contracts.md:15-18)\n is the template for the D6 AuthError codes.\n- Secret redaction in errors (docs/current-contracts.md:22) applies to the\n masked key suffix in D6.\n- Timing instrumentation and survey (docs/benchmarks.md:9-10) power D11.\n- Changelog, API reference, contributor guide, support contact exist;\n D5/D6/D7 add entries rather than new documents, except the migration guide.\n\n## TODOS.md updates\n\n- D12 (accepted): **Remove `Client.evaluate()` alias.** What: delete the\n alias and DeprecationWarning at the version named in the 2.0 changelog.\n Why: one public name per action; keeps the warning honest. Pros: clean\n surface. Cons: stragglers break at that version (the notice period is the\n mitigation). Context: alias in evalkit/client.py; remediation is\n docs/migrating-v1-to-v2.md and the codemod. Depends on: D7 shipped in\n 2.0.0b1, changelog names the removal version. Written to TODOS.md when\n plan mode exits.\n\n## DX Scorecard\n\n```\n+====================================================================+\n| DX PLAN REVIEW \u2014 SCORECARD |\n+====================================================================+\n| Dimension | Score | Prior | Trend |\n|----------------------|--------|--------|--------|\n| Getting Started | 9/10 | 3/10 | +6 \u2191 |\n| API/CLI/SDK | 8/10 | 4/10 | +4 \u2191 |\n| Error Messages | 9/10 | 5/10 | +4 \u2191 |\n| Documentation | 8/10 | 4/10 | +4 \u2191 |\n| Upgrade Path | 9/10 | 2/10 | +7 \u2191 |\n| Dev Environment | 9/10 | 9/10 | 0 = |\n| Community | 7/10 | 7/10 | 0 = |\n| DX Measurement | 9/10 | 6/10 | +3 \u2191 |\n+--------------------------------------------------------------------+\n| TTHW | ~1 min (est., gate D11) | 6 min | \u2191 |\n| Competitive Rank | Champion (pending D11 measurement) |\n| Magical Moment | designed via terminal demo (python -m evalkit.demo) |\n| Product Type | Library/SDK (Python) |\n| Mode | POLISH |\n| Overall DX | 8.5/10 | 5.0/10 | +3.5 \u2191 |\n+====================================================================+\n| DX PRINCIPLE COVERAGE |\n| Zero Friction | covered (D3, D4) |\n| Learn by Doing | covered (demo + shipped first_eval.py) |\n| Fight Uncertainty | covered (D6, D10, retained CI timeout error) |\n| Opinionated + Escape Hatches | covered (keyword-only + alias path) |\n| Code in Context | covered (first_eval.py: real dataset/evaluator)|\n| Magical Moments | covered (demo returns in seconds, D3) |\n+====================================================================+\n```\n\"Prior\" = the plan as drafted before this review. No earlier DX review exists.\nCommunity at 7 is the only dimension below 8; no adoption blocker there, and\ncommunity programs are excluded from this release by contract.\n\n## DX Implementation Checklist\n\n```\nDX IMPLEMENTATION CHECKLIST\n============================\n[ ] Time to hello world < 2 min, measured per D11\n[ ] Installation is one command (already: pip install evalkit==2.0.0b1)\n[ ] First run produces meaningful output within seconds (D3)\n[ ] Magical moment delivered via terminal demo, with next-step footer (D10)\n[ ] Every error message has: problem + cause + fix + docs link (D6 closes the last gap)\n[ ] API naming is guessable without docs (D5: one order, keyword-only)\n[ ] Every parameter has a sensible default (unchanged; no new params)\n[ ] Docs have copy-paste examples that actually work (D4)\n[ ] Examples show real use cases, not just hello world (first_eval.py)\n[ ] Upgrade path documented with migration guide (D7)\n[ ] Breaking changes have deprecation warnings + codemods (D7)\n[x] Type annotations included (existing)\n[x] Works in CI/CD without special configuration (noninteractive mode exists;\n binding check relocated there by D3)\n[x] Free tier available, no credit card required (keyless demo)\n[x] Changelog exists and is maintained (add Breaking changes block, D5/D7)\n[ ] Search works in documentation (not assessed; no docs site in scope)\n[x] Support channel exists (support contact documented)\n```\n\n## Implementation Tasks\nSynthesized from this review's findings. Each task derives from a specific\nfinding above. Run with Claude Code or Codex; checkbox as you ship.\n\n- [ ] **T1 (P1, human: ~2 days / CC: ~30 min)** \u2014 runtime first-run \u2014 Move the sample-project binding CI check off the keyless demo and first local eval; run it before the first live (keyed) evaluation and in noninteractive CI mode, keeping the existing progress line, pre-countdown explanation, EVALKIT_CI_TIMEOUT message, and help link.\n - Surfaced by: Friction D3 \u2014 docs/current-contracts.md:3-10, docs/benchmarks.md\n - Files: evalkit/demo.py, evalkit/client.py, docs/current-contracts.md, README.md:17-23\n - Verify: fresh venv, `pip install` then `python -m evalkit.demo` prints scores in under 15s with no network; first keyed `run_eval` performs the check once\n- [ ] **T2 (P1, human: ~half day / CC: ~10 min)** \u2014 quickstart + packaging \u2014 Reorder README quickstart to demo first; author and ship examples/first_eval.py in the wheel and examples archive with expected output; add a release check for its presence.\n - Surfaced by: Friction D4 \u2014 README.md:11, docs/package-contents.txt:8-9\n - Files: README.md, examples/first_eval.py, packaging manifest, docs/package-contents.txt\n - Verify: `unzip -l` of the built wheel and the examples archive both list examples/first_eval.py; `python examples/first_eval.py` runs after install\n- [ ] **T3 (P1, human: ~1 day / CC: ~15 min)** \u2014 public API \u2014 Make run_eval and run_batch `(*, dataset, evaluator)` keyword-only with a TypeError that names both parameters and the corrected call; update docs/api.md, README, example, docstrings, changelog.\n - Surfaced by: Friction D5 \u2014 docs/api.md:5-9\n - Files: evalkit/__init__.py, evalkit/client.py, docs/api.md, examples/first_eval.py, CHANGELOG\n - Verify: `run_batch(ds, ev)` raises the named TypeError; `run_batch(dataset=ds, evaluator=ev)` succeeds; api.md shows keyword form\n- [ ] **T4 (P1, human: ~1 day / CC: ~15 min)** \u2014 auth errors \u2014 AuthError carries code (MISSING_KEY / INVALID_KEY / REVOKED_KEY), cause, fix, masked key suffix, help URL; codes added to the API reference beside EVALKIT_CI_TIMEOUT.\n - Surfaced by: Friction D6 \u2014 docs/api.md:11-13\n - Files: evalkit/client.py, docs/api.md, API reference error table\n - Verify: unset key, wrong key, revoked key each produce a distinct code and message containing the console URL; full key never appears in output\n- [ ] **T5 (P1, human: ~1.5 days / CC: ~20 min)** \u2014 upgrade path \u2014 Keep Client.evaluate() as a DeprecationWarning alias for Client.run(); write docs/migrating-v1-to-v2.md (rename, run_batch order, AuthError codes, first-run change); ship codemod with fixture; changelog Breaking changes block names the removal version.\n - Surfaced by: Friction D7 \u2014 docs/api.md:15-18\n - Files: evalkit/client.py, docs/migrating-v1-to-v2.md, evalkit/codemod.py (or documented recipe), CHANGELOG\n - Verify: v1 sample script runs under 2.0.0b1 with one DeprecationWarning; codemod converts the fixture to Client.run() and keyword args\n- [ ] **T6 (P2, human: ~1 hour / CC: ~5 min)** \u2014 demo output \u2014 Append next-step footer to `python -m evalkit.demo` (console URL, export line, first_eval.py command; skip key line if EVALKIT_API_KEY is set); update README.md:34-36 and the output fixture.\n - Surfaced by: Pass 1 D10\n - Files: evalkit/demo.py, README.md\n - Verify: demo output ends with the footer; with EVALKIT_API_KEY set the key line is omitted\n- [ ] **T7 (P2, human: ~half day / CC: n/a)** \u2014 release gate \u2014 Re-run the docs/benchmarks.md onboarding protocol on the release candidate; record the measured TTHW; block the beta if 2 minutes or more.\n - Surfaced by: Pass 8 D11\n - Files: docs/benchmarks.md, release checklist\n - Verify: docs/benchmarks.md has a dated 2.0.0b1 row under 2 minutes\n- [ ] **T8 (P3, human: ~1 hour / CC: ~5 min)** \u2014 follow-up \u2014 TODOS.md entry: remove Client.evaluate() alias at the changelog's stated version.\n - Surfaced by: TODO D12\n - Files: TODOS.md\n - Verify: entry present with version, file, and remediation path\n- [ ] **T9 (P3, human: ~5 min / CC: ~1 min)** \u2014 repo setup \u2014 Append gstack skill routing section to CLAUDE.md and commit (`chore: add gstack skill routing rules to CLAUDE.md`), after plan mode exits.\n - Surfaced by: D0 onboarding prompt\n - Files: CLAUDE.md\n - Verify: `git log -1` shows the commit; CLAUDE.md ends with the Skill routing section\n\n_No new tasks from Pass 2, Pass 3, Pass 4, Pass 5 beyond T3-T5; no new tasks from Pass 6 or Pass 7._\n\n## Verification (end to end)\n\n1. Fresh virtualenv on Linux or macOS; `python -m pip install evalkit==2.0.0b1` from the release candidate index.\n2. `python -m evalkit.demo` with network disabled: three score lines plus the next-step footer, under 15 seconds.\n3. `export EVALKIT_API_KEY=bad` then `python examples/first_eval.py`: AuthError with EVALKIT_AUTH_INVALID_KEY, console URL, masked key.\n4. Valid key: first_eval.py performs the binding check once (progress lines retained), then prints a real result.\n5. v1 script calling `Client.evaluate()`: runs, emits one DeprecationWarning naming `Client.run()`.\n6. `run_batch(ds, ev)` positional: TypeError names dataset and evaluator.\n7. Stopwatch run of steps 1 to 4 by a developer who has not used the beta: under 2 minutes to first real result (D11); record in docs/benchmarks.md.\n\n## Unresolved Decisions\n\nNone. D0 through D12 were each answered.\n\n## GSTACK REVIEW REPORT\n\n| Review | Trigger | Why | Runs | Status | Findings |\n|--------|---------|-----|------|--------|----------|\n| CEO Review | `/plan-ceo-review` | Scope & strategy | 0 | \u2014 | \u2014 |\n| Outside Review | codex via `/plan-devex-review` | Independent 2nd opinion | 1 | disabled | codex_reviews disabled; no substitute reviewer run |\n| Eng Review | `/plan-eng-review` | Architecture & tests (required) | 0 | \u2014 | \u2014 |\n| Design Review | `/plan-design-review` | UI/UX gaps | 0 | \u2014 | \u2014 |\n| DX Review | `/plan-devex-review` | Developer experience gaps | 1 | clean | score: 5/10 \u2192 8.5/10, TTHW: 6 min \u2192 <2 min (est., gated by D11); 5 contract repairs + 2 polish additions |\n\n- **OUTSIDE COVERAGE:** provider codex, phase plan-review, outside_status disabled (2026-09-10T11:44:36Z). No findings; coverage intentionally absent.\n- **VERDICT:** DX CLEARED \u2014 eng review required\n\nNO UNRESOLVED DECISIONS\n", + "screen": " | REVIEW READINESS DASHBOARD |\n +====================================================================+\n | Review | Runs | Last Run | Status | Required |\n |-----------------|------|---------------------|-----------|----------|\n | Eng Review | 0 | \u2014 | \u2014 | YES |\n | CEO Review | 0 | \u2014 | \u2014 | no |\n | Design Review | 0 | \u2014 | \u2014 | no |\n | Adversarial | 0 | \u2014 | \u2014 | no |\n | Outside Voice | 1 | 2026-09-10 11:44 | DISABLED | no |\n | DX Review | 1 | 2026-09-10 11:46 | CLEAR | no |\n +--------------------------------------------------------------------+\n | VERDICT: NOT CLEARED \u2014 eng review required (you chose to run it manually) |\n +====================================================================+\n\n STATUS: DONE. Plan written to\n /tmp/gstack-paid-shard-H02ip0/tmp/gstack-e2e-plan-devex-AnQlB9/gstack-test-plan-devex.md.\n\n What the review decided, all on the recommended option:\n - Five contract repairs (T1 to T5, P1): CI check moved off the keyless demo and first local eval; quickstart reordered\n with examples/first_eval.py actually shipped; run_eval and run_batch aligned and keyword-only; AuthError gets\n codes, cause, fix, and a masked key; Client.evaluate() kept as a deprecation alias with a migration guide and\n codemod.\n - Two polish additions (T6, T7, P2): demo prints a next-step footer; the onboarding study is re-run on the release\n candidate as an under-2-minute ship gate.\n - Follow-ups (T8, T9, P3): TODO to remove the alias at the stated version; append gstack routing rules to CLAUDE.md\n and commit once plan mode exits.\n\n Scores moved from 5.0 to 8.5 overall, with TTHW estimated at about 1 minute pending the T7 measurement. Outside Codex\n review is disabled in this install and was recorded as such, not substituted. One learning logged: this repo\n pre-settles the Step 0 inputs, so future DX reviews should skip straight to the contract issues. Eng review is still\n open and left to you, as requested.\n\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n Exit plan mode?\n\n Claude wants to exit plan mode\n\n \u276f 1. Yes, and switch to default (ask each time) for this session\n 2. No\n" +} diff --git a/test/helpers/autoplan-phase-observer.ts b/test/helpers/autoplan-phase-observer.ts index cc2a64d5c..5a78e20b5 100644 --- a/test/helpers/autoplan-phase-observer.ts +++ b/test/helpers/autoplan-phase-observer.ts @@ -9,7 +9,15 @@ function phaseDeclaration(text: string): RegExpExecArray | null { const declaration = String.raw`Phase[ \t]+(1|2(?:\.5)?|3)(?:[ \t]+\(([^()]*)\))?[ \t]+(?:is[ \t]+)?(?:complete(?:d)?|done|finished|wrapped[ \t]+up)`; const plain = text.replace(new RegExp(String.raw`^\*\*(${declaration}[.:]?)\*\*`, 'i'), '$1'); if (/\bEmit\s+phase-transition\s+summary\s*:/i.test(plain)) return null; - const match = new RegExp(String.raw`^${declaration}(?:[.:](?:[ \t]+.*)?|)$`, 'i').exec(plain); + let match = new RegExp(String.raw`^${declaration}(?:[.:](?:[ \t]+.*)?|)$`, 'i').exec(plain); + if (!match) { + // A dash can separate an actual completion from its retained-work recap. + // Keep this new continuation affirmative; source, future and withdrawn + // claims cannot supply the missing phase declaration. + const dash = new RegExp(String.raw`^${declaration}[ \t]*[—–][ \t]*(.+)$`, 'i').exec(plain); + const tail = dash?.[3]?.trim(); + if (tail && !/^["“'‘>]|\?|\b(?:if|unless|when|once|pending|maybe|perhaps|would|could|will|source|example|sample|quote(?:d)?|historical|earlier|previous(?:ly)?|template|not|no|never|superseded|provided|rejected|incomplete|unfinished|withdrawn|retracted|cancelled|canceled)\b/i.test(tail)) match = dash; + } // Optional phase names are metadata, and must agree with the phase number. const names: Record = { '1': /^CEO(?:[ \t]+review)?$/i, diff --git a/test/helpers/autoplan-setup-question.ts b/test/helpers/autoplan-setup-question.ts index 9cc8b3eb0..17b9295e5 100644 --- a/test/helpers/autoplan-setup-question.ts +++ b/test/helpers/autoplan-setup-question.ts @@ -1,6 +1,7 @@ import { capturePlanCountQuestion, parseNumberedOptions, planCountPrerequisitePick, planCountQuestionInput, planCountSubmissionInput, type AskUserQuestionFingerprint } from './claude-pty-runner'; -import type { NativePlanQuestion, NativePlanQuestionCall } from './plan-count-transcript'; +import type { NativePlanQuestion, NativePlanQuestionCall, NativePublicToolEvent, PlanCountTranscript } from './plan-count-transcript'; +import type { readPendingQuestion } from './plan-count-pending-question'; /** A copied native panel is not actionable after prose or inside a code example. */ function activeSetupPanel(visible: string): boolean { @@ -365,3 +366,52 @@ export function autoplanRoutingSetupInput(visible: string, seen: Set, pe for (const signature of decision.signatures) seen.add(signature); return decision.input; } + +/** Identify a remaining native human wait. The caller must treat it as failure, never phase credit. */ +export function autoplanBlockingQuestionBoundary(visible: string, context: { + commandStartedAt: number; viewportCapturedAt: number; + /** Both projections must come from the same owned readPlanCountTranscript poll. */ + transcript: PlanCountTranscript; publicTools: NativePublicToolEvent[]; + /** Only the current return from the owned, post-command readPendingQuestion. */ + pending?: ReturnType; +}): { sessionId: string; toolUseId: string; source: 'native' | 'pre_tool_use' } | null { + const {transcript, commandStartedAt, viewportCapturedAt, publicTools} = context; + if (transcript.status !== 'ready' || !Number.isFinite(commandStartedAt) || + !Number.isFinite(viewportCapturedAt) || commandStartedAt < 0 || viewportCapturedAt < commandStartedAt) return null; + const sessions = new Set([...transcript.calls.map(call => call.sessionId), + ...transcript.assistantMessages.map(message => message.sessionId)]); + if (sessions.size !== 1 || ![...sessions][0]) return null; + const unanswered = transcript.calls.filter(call => !call.answered && !call.failed); + if (unanswered.length > 1) return null; + const call = unanswered[0] ?? context.pending; + if (!call || !sessions.has(call.sessionId) || !call.toolUseId || call.answered !== false || + call.failed !== false || call.questions.length !== 1 || call.questions[0]!.multiSelect) return null; + const identity = (questions: unknown): string | null => Array.isArray(questions) && questions.every(q => + q && typeof q.header === 'string' && typeof q.question === 'string' && Array.isArray(q.options) && + q.options.every((o: any) => o && typeof o.label === 'string' && + (o.description === undefined || typeof o.description === 'string'))) + ? JSON.stringify(questions.map(q => [q.header,q.question,q.multiSelect ?? false, + q.options.map((o: any) => [o.label,o.description ?? ''])])) : null; + if (identity(call.questions) === null) return null; + const uses = publicTools.filter(event => event.sessionId === call.sessionId && event.toolUseId === call.toolUseId && event.kind === 'use'); + if (publicTools.some(event => event.sessionId === call.sessionId && event.toolUseId === call.toolUseId && event.kind === 'result')) return null; + let source: 'native' | 'pre_tool_use'; + if (unanswered.length) { + const use = uses[0], at = Date.parse(use?.timestamp ?? ''); + if (uses.length !== 1 || use?.name !== 'AskUserQuestion' || !Number.isFinite(at) || + at < commandStartedAt || at > viewportCapturedAt || !Array.isArray(use.input?.questions) || + identity(use.input.questions as NativePlanQuestion[]) !== identity(call.questions)) return null; + source = 'native'; + } else { + // The reader has already checked cwd/config, parent session, timestamp, + // recorder poison/lock state and absence of a published result or call. + if (context.pending?.source !== 'pre_tool_use' || uses.length || + transcript.calls.some(row => row.sessionId === call.sessionId && row.toolUseId === call.toolUseId)) return null; + source = 'pre_tool_use'; + } + const display = visible.replace(/(^|[\r\n])[\t ]*[│┃][\t ]?/g, '$1'); + const headerAt = display.search(/^ {0,3}[☐□]/m); + if (headerAt < 0 || /^(?:Source|Example|Quoted|Historical|Template)\b[^\n]*:/im.test(display.slice(0, headerAt)) || + !completeSetupOptions(display, call)) return null; + return {sessionId:call.sessionId,toolUseId:call.toolUseId,source}; +} diff --git a/test/helpers/claude-pty-runner.ts b/test/helpers/claude-pty-runner.ts index ce60c575e..20f10be11 100644 --- a/test/helpers/claude-pty-runner.ts +++ b/test/helpers/claude-pty-runner.ts @@ -1956,7 +1956,7 @@ function isCompletedDxHandoff(call: NativePlanQuestionCall): boolean { if (!call.answered || call.failed || call.questions.length !== 1 || !Array.isArray(call.unansweredQuestionIndices) || call.unansweredQuestionIndices.length) return false; const q = call.questions[0]!; - if (resolvedDxTaskNavigation(call) || specifiedDxTaskNavigation(call) || architecturalDxTaskNavigation(call)) return true; + if (manualDxTaskNavigation(call) || resolvedDxTaskNavigation(call) || specifiedDxTaskNavigation(call) || architecturalDxTaskNavigation(call)) return true; const header = q.header.trim().replace(/^D\s*\d+\s*(?:[—–:-]\s*)?/i, ''); const question = q.question.replace(/^D\s*\d+\s*[—–:-]\s*/i, ''); const completionLines: string[] = []; @@ -2025,6 +2025,43 @@ function isCompletedDxHandoff(call: NativePlanQuestionCall): boolean { q.options.some(o => o.label === call.answers?.[q.question]); } +/** A selected manual handoff leaves already recorded DX decisions unchanged. */ +function manualDxTaskNavigation(call: NativePlanQuestionCall): boolean { + const q = call.questions[0]!; + if (call.answered !== true || call.failed !== false || !call.sessionId || !call.toolUseId || + !Number.isFinite(Date.parse(call.answeredAt ?? '')) || q.multiSelect || + q.header.trim() !== 'Next steps' || q.options.length !== 3 || + new Set(q.options.map(o => o.label)).size !== 3 || Object.keys(call.answers ?? {}).length !== 1) return false; + const prose = (text: string) => text.split(/\r?\n/).filter(line => + !/^\s*>/.test(line) && !/^\s*(["'`]).*\1\s*$/.test(line)).join('\n').trim(); + const text = prose(q.question); + if (/```|~~~||→) (?:10|[0-9])(?:\.\d+)?\/10\))?\. (?:What should happen next|What happens next|What['’]s next)\?\n/i.test(text) || + !/(?:^|\n)ELI10:\s*The DX review (?:found|identified)\b/i.test(text) || + !/(?:^|[.!]\s+)(?:All (?:are|have been) (?:written|recorded) (?:in|into) the plan as tasks\b|All DX decisions and tasks are recorded in the plan\.)/i.test(text)) return false; + const metadata = /(?:^|\n)Project\/branch\/task:\s*([^\n]*)/i.exec(text)?.[1]; + if (metadata && /^(?:if|unless|when|once|after|provided|proposed|optional|source|example|historical|earlier review|previously)\b/i.test(metadata)) return false; + const label = (value: string) => value.trim().replace(/\s*\((?:recommended|required gate)\)\s*$/i, ''); + const manual = (value: string) => /^Skip(?:,|\s*[—–-])\s*I['’]ll handle next steps manually$/i.test(label(value)); + const eng = (value: string) => /^Run \/plan-eng-review next$/i.test(label(value)); + const implement = (value: string) => /^Ready to implement(?:,|\s*[—–-])\s*run \/devex-review after shipping$/i.test(label(value)); + if (q.options.filter(o => manual(o.label)).length !== 1 || q.options.filter(o => eng(o.label)).length !== 1 || + q.options.filter(o => implement(o.label)).length !== 1) return false; + const selected = q.options.find(o => o.label === call.answers?.[q.question]); + if (!selected || !manual(selected.label)) return false; + const description = prose(selected.description ?? '').replace(/[✅❌]/g, '').trim(); + if (!/^(?:Matches your stated intent\b|Plan exits now\b|Exit the plan now\b)/i.test(description) || + !/(?:^|[.!]\s+)(?:Plan exits now|Exit the plan now) with all DX (?:decisions and tasks|tasks and decisions) recorded[;.]\s*(?:nothing else is started|no further review is started)\./i.test(description)) return false; + // A completed recap cannot conceal another plan decision or an instruction + // to change the plan before the selected manual exit. Quoted archive lines + // do not establish current obligations; direct current-status quotes do. + const current = `${text}\n${description}`.replace(/\byou will run subsequent reviews yourself\b/gi, 'manual follow-up'); + if (/(?:^|\n|[.!;]\s+)(?:Source|Example|Historical(?: review)?|Previously|Earlier review(?: assessment)?):/i.test(current)) return false; + if (/\b(?:This|The) (?:manual |DX )?handoff (?:is|has been) [\"'`]?(?:cancell?ed|withdrawn|retracted|superseded|not current)\b/i.test(current)) return false; + return !/\b(?:unresolved|outstanding)\b|\b(?:not|never)\s+(?:all\s+)?(?:done|complete|completed|recorded|written|resolved)\b|\b(?:review|findings?|decisions?|tasks?|plan)\b[^.!?\n]{0,70}\b(?:pending|remaining|withdrawn|retracted|superseded)\b|\b(?:only|complete)\s+(?:after|if|when|once)\b/i.test(current) && + !/(?:^|[.!?;]\s+|\n|\b(?:should|must|need to|will)\s+)(?:(?:we|you|please|first|then|also)\s+)*(?:add|fix|edit|update|rewrite|remove|implement|resolve|decide|change|approve|start|run)\b/im.test(current); +} + /** The next gate may validate architecture already decided by this DX review. */ function architecturalDxTaskNavigation(call: NativePlanQuestionCall): boolean { const q = call.questions[0]!; @@ -2415,7 +2452,8 @@ function nativeExplicitCeoFinding(fp: AskUserQuestionFingerprint, allowQuestionI // heading. Sentence punctuation and the form of the remedy question do // not change the completed brief's current defect and offered amendment. const testAssertion = /^Test ([1-9]\d*)(?:\s+\([^()\n]*\))?\s+(?:asserts?|checks?)\s+only\b[^\n]+\?$/i.exec(normalized); - if (testAssertion && !/^(?:finding|issue)\b|^f\d/i.test(q.header.trim())) { + const testIdentity = testAssertion ?? /^Test ([1-9]\d*)(?:\s+\([^()\n]*\))?(?:\s*[:—–-]\s*|\s+)[^\n]+\?$/i.exec(normalized); + if (testIdentity && !/^(?:finding|issue)\b|^f\d/i.test(q.header.trim())) { if ((fp.nativeQuestionIndex !== undefined && fp.nativeQuestionIndex !== 0) || typeof call.answeredAt !== 'string' || !Number.isFinite(Date.parse(call.answeredAt))) return false; const headerTest = /^Test\s+([1-9]\d*)\b/i.exec(q.header.trim()); @@ -2446,8 +2484,30 @@ function nativeExplicitCeoFinding(fp: AskUserQuestionFingerprint, allowQuestionI !description.split(/[.!?]\s+|\n/).some(clause => /^(?:Correction:\s*)?(?:this|that|the)\s+(?:amendment|remedy|option|decision)\s+(?:is|has been)\s+(?:(?:only|just|an?)\s+)*(?:withdrawn|retracted|rejected|cancelled|canceled|not current|historical|hypothetical|quoted|source|example)\b/i.test(clause.trim())); }); - if ((!headerTest || headerTest[1] === testAssertion[1]) && ownedPrefix && !framed && !conditionalContext && !withdrawn && - currentAssessment && currentAmendment && ceoNumberedBriefDecision(q, normalized)) return true; + let reviewSubject = normalized; + if (!testAssertion) { + // A Test identity can ask for its assertion without restating the + // defect in its title. Normalize only the current owned ELI10 clause; + // retain the original title so source/competing identities stay visible. + const clauses = explanation.split(/(?<=[.!?])\s+/); + const assertion = /^(?:But\s+)?(?:the\s+)?(?:planned|proposed|current)\s+test\s+only\s+checks?\s+(.+)$/i; + const at = clauses.findIndex(clause => assertion.test(clause.trim())); + const decision = /^D([1-9]\d*)\s*[—–-]/i.exec(title); + const recommended = /^Recommendation:\s*([1-9]\d*)?[A-Z]\b/im.exec(q.question); + const headerIdentity = /^Test\s+([1-9]\d*)(?=\s|[:—–-]|$)/i.exec(q.header.trim()); + const titleIdentities = normalized.replace(/"(?:[^"\\]|\\.)*"|“[^”]*”|`[^`]*`/g, '""') + .matchAll(/\bTest\s+(\d+(?:\.\d+)*)\b/gi); + if ((/^Test\s+\d/i.test(q.header.trim()) && (!headerIdentity || headerIdentity[1] !== testIdentity[1])) || + (/^D\d/i.test(title) && !decision) || + [...titleIdentities].some(identity => identity[1] !== testIdentity[1])) return false; + if (at < 0 || clauses.slice(0, at + 1).some(clause => !ceoCurrentBriefProse(clause)) || + !ceoCurrentBriefProse(q.question, false) || /\b(?:Finding|Issue)\s+F?[1-9]\d*/i.test(normalized) || + (decision && recommended?.[1] && decision[1] !== recommended[1])) return false; + reviewSubject += ` Test ${testIdentity[1]} checks only ${assertion.exec(clauses[at]!.trim())![1]}`; + } + if ((!headerTest || headerTest[1] === testIdentity[1]) && ownedPrefix && !framed && !conditionalContext && !withdrawn && + currentAssessment && currentAmendment && ceoNumberedBriefDecision(q, reviewSubject, !testAssertion, + testAssertion ? undefined : option => ceoCurrentBriefProse(option.label.replace(/^(?:[1-9]\d*)?[A-Z][):.]\s*/i, '')) && ceoCurrentBriefProse(option.description ?? ''))) return true; } // Section and finding counters identify a brief; they cannot supply its // current assessment or the authority of an offered amendment. @@ -3471,6 +3531,10 @@ export async function runPlanSkillObservation(opts: { cwd?: string; /** Total budget for skill to reach a terminal outcome. Default 180000. */ timeoutMs?: number; + /** Keep observing a judge-only waiting verdict when the caller requires + * a rendered prose choice list. Deterministic terminal outcomes retain + * precedence; this does not grant prose credit to a judge verdict. */ + requireProseEvidence?: boolean; /** Extra CLI args appended after --permission-mode. Used by the v1.22+ * AskUserQuestion-blocked regression tests to pass * `['--disallowedTools', 'AskUserQuestion']` (the flag set Conductor @@ -3696,6 +3760,7 @@ export async function runPlanSkillObservation(opts: { lastJudgeVerdict = judgePtyState(visible, { testName: opts.skillName }); if (lastJudgeVerdict.state === 'waiting' && !pendingSeededCompletion) { waitingEverObserved = true; + if (opts.requireProseEvidence && !proseAUQEverObserved) continue; return { outcome: 'asked', summary: `LLM judge: ${lastJudgeVerdict.reasoning} (state=waiting after ${Math.round(elapsed / 1000)}s)`, @@ -3713,7 +3778,7 @@ export async function runPlanSkillObservation(opts: { // by the time the timeout fires, the buffer has moved past the // options into spinner state but the question DID surface earlier. const finalVisible = session.visibleSince(since); - if (proseAUQEverObserved || waitingEverObserved) { + if (proseAUQEverObserved || waitingEverObserved && !opts.requireProseEvidence) { return { outcome: 'asked', summary: diff --git a/test/helpers/design-count-review.ts b/test/helpers/design-count-review.ts index 00e7c11aa..c9dc42812 100644 --- a/test/helpers/design-count-review.ts +++ b/test/helpers/design-count-review.ts @@ -78,21 +78,23 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean { // A source citation alone can describe a report or the next reviewer. // Bind the alternate wording to a named control's concrete style amendment // and the opposed choice that leaves the documented violation unresolved. - const primary = /^Make ([A-Za-z][A-Za-z0-9 _-]{0,39}) the (?:visible|(?:only|single)(?: filled)?) primary action(?: in the header)?$/i.exec(issue[2]!) ?? + const primary = /^Make ([A-Za-z][A-Za-z0-9 _-]{0,39}) the (?:visible|visually|(?:only|single)(?: filled)?) primary action(?: in the header)?$/i.exec(issue[2]!) ?? /^How should (?:the )?(?:header )?actions establish that ([A-Za-z][A-Za-z0-9 _-]{0,39}) is the primary action$/i.exec(issue[2]!); const explicitStyle = primary && `${primary[1]} filled (?:primary )?#[0-9a-f]{6}(?:/| with )(?:white|black)(?: text)?; ` + '[A-Za-z][A-Za-z0-9 ,/_-]{0,99} neutral ghost(?: buttons)?\\.'; const amendments = primary && [ new RegExp(`^(?:✅\\s*)?Matches DESIGN\\.md exactly: ${primary[1]} filled #[0-9a-f]{6} with (?:white|black) text; ` + '[A-Za-z][A-Za-z0-9 ,_-]{0,99} as neutral ghost buttons\\.', 'i'), - new RegExp(`^(?:✅\\s*)?${primary[1]} becomes the single filled primary \\(#[0-9a-f]{6}, (?:white|black) text\\); ` + - '[A-Za-z][A-Za-z0-9 /,_-]{0,99} become neutral ghost buttons exactly as DESIGN\\.md specifies\\b', 'i'), + new RegExp(`^(?:✅\\s*)?${primary[1]} becomes the (?:single|one) filled(?: primary)?(?: button)? \\(#[0-9a-f]{6}, (?:white|black) text\\); ` + + '[A-Za-z][A-Za-z0-9 /,_-]{0,99} become neutral ghost buttons (?:exactly as DESIGN\\.md specifies|per DESIGN\\.md)\\b', 'i'), new RegExp(`^(?:✅\\s*)?${primary[1]} is the single filled #[0-9a-f]{6} button; ` + '[A-Za-z][A-Za-z0-9 /,_-]{0,99} become neutral ghosts, exactly per DESIGN\\.md\\b', 'i'), new RegExp(`^(?:✅\\s*)?Apply DESIGN\\.md tokens: ${primary[1]} #[0-9a-f]{6} filled with (?:white|black) text; ` + '[A-Za-z][A-Za-z0-9 ,/_-]{0,99} neutral ghost(?: buttons)?\\.', 'i'), // The same concrete style can cite DESIGN.md before or after its tokens. new RegExp(`^(?:✅\\s*)?(?:Apply DESIGN\\.md(?: tokens)?: ${explicitStyle}|${explicitStyle} Exact DESIGN\\.md\\.)`, 'i'), + new RegExp(`^(?:✅\\s*)?${primary[1]}\\s*=\\s*filled #[0-9a-f]{6} with (?:white|black) text; ` + + '[A-Za-z][A-Za-z0-9 ,/_-]{0,99}\\s*=\\s*neutral ghost(?: buttons?)?,? per DESIGN\\.md\\b', 'i'), ]; const primaryHeader = !q.header.includes(':') || q.header.split(':')[1]!.trim().toLowerCase() === primary?.[1]?.toLowerCase(); // The style wordings share one owned decision: a current equal-weight gap, @@ -104,8 +106,8 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean { // A quoted status scalar remains a current assertion when its unquoted // subject names this decision; whole quoted historical prose stays absent. .replace(/"[^"\n]*"|“[^”\n]*”/g, (quoted: string, index: number, source: string) => - /^(?:withdrawn|resolved|closed|hypothetical|rejected|cancelled|canceled|not current|no longer current)$/i.test(quoted.slice(1, -1)) && - /(?:^|[.!?;]\s+|\n)(?:Correction:\s*)?(?:(?:This (?:issue|finding|question|amendment|deferral|style|fix|remedy|choice|option)|Issue [1-9]\d*) (?:is|was|has been)|(?:these|the|this) (?:tokens?|styles?|primary treatment) (?:are|is|were|was|have been|has been)) $/i.test(source.slice(0, index)) + /^(?:withdrawn|superseded|resolved|closed|hypothetical|rejected|cancelled|canceled|not current|no longer current)$/i.test(quoted.slice(1, -1)) && + /(?:^|[.!?;]\s+|\n)(?:Correction:\s*)?(?:(?:This (?:issue|finding|question|amendment|deferral|style|fix|remedy|choice|option|(?:DESIGN\.md |token )?(?:requirement|contract))|Issue [1-9]\d*) (?:is|was|has been)|(?:these|the|this) (?:tokens?|styles?|primary treatment) (?:are|is|were|was|have been|has been)) $/i.test(source.slice(0, index)) ? quoted.slice(1, -1) : '').replace(/\*\*/g, ''); const questionText = currentText(q.question); const assessments = [...questionText.matchAll(/^ELI10: (.+)$/gm)]; @@ -132,7 +134,7 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean { const variantContract = primary && new RegExp(`(?:^|[.!?]\\s+)DESIGN\\.md already says ${primary[1]} is the only filled button ` + '\\(#[0-9a-f]{6} with (?:white|black) text(?:, about [0-9]+(?:\\.[0-9]+)?:1 contrast)?\\) and the other ' + '(two|three|four|five|six|seven|eight|nine|ten|[1-9]\\d*) are neutral ghost buttons\\.', 'i').exec(assessment); - const invalidContract = /(?:^|[.!?]\s+|\n)(?:Correction:\s*)?(?:this|that|the) (?:(?:DESIGN\.md|token) )?(?:requirement|contract) (?:is|was|has been) (?:withdrawn|superseded|not current|no longer current)\b/i; + const invalidContract = /(?:^|[.!?]\s+|\n)(?:Correction:\s*)?(?:this|that|the) (?:(?:DESIGN\.md|token) )?(?:requirement|contract) (?:is|was|has been) (?:withdrawn|superseded|rejected|cancelled|canceled|not current|no longer current)\b/i; const namedContract = primary && new RegExp(`(?:^|[.!?]\\s+)DESIGN\\.md already says ${primary[1]} is the only filled primary button and the other (two|three|four|five|six|seven|eight|nine|ten|[1-9]\\d*) are neutral ghost buttons\\.`, 'i').exec(assessment); const statedVariant = (variantContract || namedContract) && numberValue((variantContract || namedContract)![1]!) === otherControls && @@ -146,7 +148,7 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean { const primaryRepair = primaryHeader && amendments && currentPrimary && !!call.answeredAt && Number.isFinite(Date.parse(call.answeredAt)) && choiceIds.every(id => id?.[1] === issue[1]) && - !withdrawn.test(questionText) && !closedGap.test(questionText) && !withdrawnStyles.test(questionText) && + !withdrawn.test(questionText) && !closedGap.test(questionText) && !withdrawnStyles.test(questionText) && !invalidContract.test(questionText) && q.options.some(amendment => { const body = currentText(amendment.description ?? ''); const style = amendments.map(pattern => pattern.exec(body)).find(Boolean)?.[0]; @@ -171,14 +173,17 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean { // Native menus can list current benefits before the gap retained by // declining. Only consume a complete affirmative pro/con prefix; prose // framing a source example or a future condition cannot expose an icon. - const pros = /^(?:✅(?!\s*(?:If|When|Unless|Historical|Hypothetical|Quoted|Source|Example)\b)\s*[^✅❌]+)+❌\s*/i.exec(declined); - const remaining = pros && !sourceAssessment.test(pros[0]) ? declined.slice(pros[0].length) : declined; + const headerDeferral = /^(?:Leave|Keep) the header unchanged and record the gap as (?:debt|an open issue)\.\s*/i.exec(declined); + const deferralBody = headerDeferral ? declined.slice(headerDeferral[0].length) : declined; + const cancelledHeaderDeferral = headerDeferral && /(?:^|[.!?;]\s+|\n)(?:Correction:\s*)?(?:do not|don't|never|skip|cancel|withdraw) (?:keep|leave) (?:the )?header unchanged\b/i.test(declined); + const pros = /^(?:✅(?!\s*(?:If|When|Unless|Historical|Hypothetical|Quoted|Source|Example)\b)\s*[^✅❌]+)+❌\s*/i.exec(deferralBody); + const remaining = pros && !sourceAssessment.test(pros[0]) ? deferralBody.slice(pros[0].length) : deferralBody; const retainedButtons = /^(?:❌\s*)?Keep all (two|three|four|five|six|seven|eight|nine|ten|[1-9]\d*) (?:header )?buttons identical; gap stays documented\./i.exec(remaining); const cancelledRetainedButtons = /(?:^|[.!?;]\s+|\n)(?:Correction:\s*)?(?:do not|don't|never|skip|cancel|withdraw) (?:keep|leave) (?:all )?(two|three|four|five|six|seven|eight|nine|ten|[1-9]\d*) (?:header )?buttons identical\b/i.exec(declined); - return defer !== amendment && !sourceAssessment.test(declined) && !withdrawn.test(declined) && !closedGap.test(declined) && + return defer !== amendment && !sourceAssessment.test(declined) && !withdrawn.test(declined) && !closedGap.test(declined) && !cancelledHeaderDeferral && ((retainedButtons && numberValue(retainedButtons[1]!) === otherControls + 1 && (!cancelledRetainedButtons || numberValue(cancelledRetainedButtons[1]!) !== otherControls + 1)) || - /^(?:❌\s*)?(?:Leaves a documented DESIGN\.md violation in place|Ships the documented violation;[^.\n]*\bthe gap remains open|Primary-action ambiguity ships; documented DESIGN\.md violation remains|Decline the fix; gap stays documented and lowers the score|Keep all (?:two|three|four|five|six|seven|eight|nine|ten|[1-9]\d*) identical; record as an open DESIGN\.md violation)\b/i.test(remaining) || + /^(?:❌\s*)?(?:Leaves a documented DESIGN\.md violation in place|Ships a (?:known|documented) DESIGN\.md violation and the primary action (?:stays|remains) undiscoverable|Ships a header with no primary action; PLAN\.md['’]s own gap stays open|Ships the documented violation;[^.\n]*\bthe gap remains open|Primary-action ambiguity ships; documented DESIGN\.md violation remains|Decline the fix; gap stays documented and lowers the score|Keep all (?:two|three|four|five|six|seven|eight|nine|ten|[1-9]\d*) identical; record as an open DESIGN\.md violation)\b/i.test(remaining) || (variantRepair && /^(?:Violates DESIGN\.md and leaves users guessing which action is primary; Pass [1-7] stays at [0-9](?:\.[0-9]+)?\/10|Documented DESIGN\.md violation ships and Pass [1-7] stays at [0-9](?:\.[0-9]+)?\/10)\.$/i.test(remaining))); }); }); diff --git a/test/helpers/plan-scope-selection.ts b/test/helpers/plan-scope-selection.ts index b4b4e391b..31afa90cb 100644 --- a/test/helpers/plan-scope-selection.ts +++ b/test/helpers/plan-scope-selection.ts @@ -22,7 +22,10 @@ function withdrawsPlanSelection(message: string, title: string): boolean { for (const statement of assertions.join('\n').split(/(?<=[.!?])\s+|\n+/).map(line => line.trim())) { if (statement.endsWith('?')) continue; const claim = statement.replace(/^Correction:\s*/i, ''); - const plain = claim.replace(/"[^"\n]*"|“[^”\n]*”|`[^`\n]*`/g, '[quoted]'); + const plain = claim.replace(/"[^"\n]*"|“[^”\n]*”|`[^`\n]*`/g, (quoted, index) => + /^(?:withdrawn|retracted|cancelled|canceled|hypothetical)$/i.test(quoted.slice(1, -1)) && + /^(?:The|This|That|My)\s+(?:(?:scope|target)\s+)?(?:selection|declaration)\s+(?:is|was|has been|remains)\s+(?:now\s+)?$/i.test(claim.slice(0, index)) + ? quoted.slice(1, -1) : '[quoted]'); if (/^(?:The|This|That|My)\s+(?:(?:scope|target)\s+)?(?:selection|declaration)\s+(?:is|was|has been|remains)\s+(?:now\s+)?(?:withdrawn|retracted|cancelled|canceled|hypothetical)\b/i.test(plain) || /^(?:(?:I|We)\s+(?:have\s+)?)?(?:withdrawn?|withdrew|retract(?:ed)?|cancel(?:led|ed)?|disregard(?:ed)?|ignore(?:d)?)\s+(?:this|that|the|my)\s+(?:selection|declaration)\b/i.test(plain)) return true; const changedTarget = /^(?:The|This|My)\s+(?:selected|review)\s+target\s+is\s+(?:now\s+)?(.+?)[.!?]?$/i.exec(claim); @@ -66,14 +69,21 @@ export function nativeSeededPlanSelection( const line = message.text.split(/\r?\n/).find(value => value.trim()); if (!line || /^(?: {4}|\t)/.test(line)) continue; const text = line.trim(); + const automatic = /^(?:I\'ll|I will) auto[- ]select option B and review\s+(?:the\s+)?(.+?)\s+(?:draft(?:\s+plan)?|plan)\s+(?:you shared|you pasted|pasted here)(.*)$/i.exec(text); + const automaticTarget = automatic?.[1]?.replace(/^(?:"([^"\n]+)"|“([^”\n]+)”|`([^`\n]+)`)$/, (_, straight, curly, code) => straight ?? curly ?? code); const selectedNow = /^(?:I've|I have) selected (?:option B, )?(?:reviewing|to review)\s+(?:the\s+)?pasted\s+(?:"([^"\n]+)"|“([^”\n]+)”|`([^`\n]+)`)\s+(?:draft(?:\s+plan)?|plan)(.*)$/i.exec(text); const selected = selectedNow ?? /^(?:Scope gate confirms plan mode, so )?(?:I'll review|I will review|I'll go with reviewing|I will go with reviewing|I'll proceed with reviewing|I will proceed with reviewing|I'm proceeding with reviewing|I am proceeding with reviewing)\s+(?:the\s+)?(?:pasted\s+)?(?:"([^"\n]+)"|“([^”\n]+)”|`([^`\n]+)`)\s+(?:draft(?:\s+plan)?|plan)(?:\s+(?:you pasted|pasted here))?(.*)$/i.exec(text); - if (!selected || (selected[1] ?? selected[2] ?? selected[3])!.trim().toLowerCase() !== title.toLowerCase()) continue; - const tail = selected[4]!; + const selectedTarget = automaticTarget ?? (selected && (selected[1] ?? selected[2] ?? selected[3])); + if (!selectedTarget || selectedTarget.trim().toLowerCase() !== title.toLowerCase()) continue; + const tail = automatic?.[2] ?? selected![4]!; if (/\b(?:if|unless|assuming|pending|only after|instead|not|won't|cannot)\b/i.test(tail)) continue; if (/\b(?:retract|withdraw|cancel|disregard|ignore)\s+(?:that|this|the|my)\s+(?:selection|declaration)\b/i.test(tail)) continue; if (/\b(?:treat|consider|regard)\s+(?:that|this|the|my)\s+(?:selection|declaration)\s+as\s+(?:a\s+)?(?:hypothetical|example|proposal)\b/i.test(tail)) continue; if (/\b(?:that|this|the|my)\s+(?:selection|declaration)\s+(?:is|was)\s+(?:withdrawn|cancelled|canceled|hypothetical|retracted)\b/i.test(tail)) continue; + if (automatic) { + if (/^(?:\.|,\s*running\s+(?:the\s+)?(?:pre-review\s+)?audit\b[^?]*\.)$/i.test(tail) && remainsSelected(message.timestamp)) return true; + continue; + } if (selectedNow) { // A completed selection may name the pasted target before its plan-mode // reason. Keep the first assertion bound; later work is not a new target. diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index a53d1775d..d59e6808f 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -103,6 +103,7 @@ export const E2E_TOUCHFILES: Record = { 'plan-ceo-review-plan-mode': [ 'test/auto-decide-saved-ai.test.ts', 'test/fixtures/auto-decide-saved-ai.json', 'test/fixtures/auto-decide-retry-ai.json','bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/helpers/plan-scope-selection.ts', 'test/plan-scope-selection.test.ts', 'test/helpers/native-auto-decide.ts', 'test/native-auto-decide.test.ts', 'test/native-auto-decide-pty.test.ts', 'test/fixtures/native-auto-decide-ag.json', 'test/eng-seeded-completion-ai.test.ts', 'test/fixtures/eng-seeded-completion-ai.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/design-scope-declaration-ak.test.ts', + 'test/design-scope-announcement-ao.test.ts', 'test/fixtures/design-scope-announcement-ao.json', 'test/fixtures/design-scope-declaration-ak.json', "test/eng-option-b-scope-al.test.ts", "test/fixtures/eng-option-b-scope-al.json", @@ -110,6 +111,7 @@ export const E2E_TOUCHFILES: Record = { 'plan-eng-review-plan-mode': [ 'test/auto-decide-saved-ai.test.ts', 'test/fixtures/auto-decide-saved-ai.json', 'test/fixtures/auto-decide-retry-ai.json','bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/helpers/plan-scope-selection.ts', 'test/plan-scope-selection.test.ts', 'test/fixtures/design-plan-scope-ag.json', 'test/design-scope-selection-aj.test.ts', 'test/fixtures/design-scope-selection-aj.json', 'test/helpers/native-auto-decide.ts', 'test/native-auto-decide.test.ts', 'test/native-auto-decide-pty.test.ts', 'test/fixtures/native-auto-decide-ag.json', 'test/eng-seeded-completion-ai.test.ts', 'test/fixtures/eng-seeded-completion-ai.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/design-scope-declaration-ak.test.ts', + 'test/design-scope-announcement-ao.test.ts', 'test/fixtures/design-scope-announcement-ao.json', 'test/fixtures/design-scope-declaration-ak.json', "test/eng-option-b-scope-al.test.ts", "test/fixtures/eng-option-b-scope-al.json", @@ -117,6 +119,7 @@ export const E2E_TOUCHFILES: Record = { 'plan-design-review-plan-mode': [ 'test/auto-decide-saved-ai.test.ts', 'test/fixtures/auto-decide-saved-ai.json', 'test/fixtures/auto-decide-retry-ai.json','bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts', 'test/skill-e2e-design.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/helpers/plan-scope-selection.ts', 'test/plan-scope-selection.test.ts', 'test/fixtures/design-plan-scope-ag.json', 'test/design-scope-selection-aj.test.ts', 'test/fixtures/design-scope-selection-aj.json', 'test/helpers/native-auto-decide.ts', 'test/native-auto-decide.test.ts', 'test/native-auto-decide-pty.test.ts', 'test/fixtures/native-auto-decide-ag.json', 'test/eng-seeded-completion-ai.test.ts', 'test/fixtures/eng-seeded-completion-ai.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/design-scope-declaration-ak.test.ts', + 'test/design-scope-announcement-ao.test.ts', 'test/fixtures/design-scope-announcement-ao.json', 'test/fixtures/design-scope-declaration-ak.json', "test/eng-option-b-scope-al.test.ts", "test/fixtures/eng-option-b-scope-al.json", @@ -124,6 +127,7 @@ export const E2E_TOUCHFILES: Record = { 'plan-devex-review-plan-mode': [ 'test/auto-decide-saved-ai.test.ts', 'test/fixtures/auto-decide-saved-ai.json', 'test/fixtures/auto-decide-retry-ai.json','bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-devex-plan-mode.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/helpers/plan-scope-selection.ts', 'test/plan-scope-selection.test.ts', 'test/helpers/native-auto-decide.ts', 'test/native-auto-decide.test.ts', 'test/native-auto-decide-pty.test.ts', 'test/fixtures/native-auto-decide-ag.json', 'test/eng-seeded-completion-ai.test.ts', 'test/fixtures/eng-seeded-completion-ai.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/design-scope-declaration-ak.test.ts', + 'test/design-scope-announcement-ao.test.ts', 'test/fixtures/design-scope-announcement-ao.json', 'test/fixtures/design-scope-declaration-ak.json', "test/eng-option-b-scope-al.test.ts", "test/fixtures/eng-option-b-scope-al.json", @@ -137,6 +141,7 @@ export const E2E_TOUCHFILES: Record = { 'plan-mode-no-op': [ 'test/auto-decide-saved-ai.test.ts', 'test/fixtures/auto-decide-saved-ai.json', 'test/fixtures/auto-decide-retry-ai.json','bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-mode-no-op.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/helpers/plan-scope-selection.ts', 'test/plan-scope-selection.test.ts', 'test/helpers/native-auto-decide.ts', 'test/native-auto-decide.test.ts', 'test/native-auto-decide-pty.test.ts', 'test/fixtures/native-auto-decide-ag.json', 'test/eng-seeded-completion-ai.test.ts', 'test/fixtures/eng-seeded-completion-ai.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/design-scope-declaration-ak.test.ts', + 'test/design-scope-announcement-ao.test.ts', 'test/fixtures/design-scope-announcement-ao.json', 'test/fixtures/design-scope-declaration-ak.json', "test/eng-option-b-scope-al.test.ts", "test/fixtures/eng-option-b-scope-al.json", @@ -167,6 +172,7 @@ export const E2E_TOUCHFILES: Record = { // the PreToolUse hook denies the flaky tool). Touches the resolver that owns // the Conductor rule, the preamble signal, the hook, and the detection helper. 'conductor-prose': [ + 'test/conductor-prose-observation-ao.test.ts', 'test/fixtures/conductor-prose-ao.json', 'test/auto-decide-saved-ai.test.ts', 'test/fixtures/auto-decide-saved-ai.json', 'test/fixtures/auto-decide-retry-ai.json','bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble.ts', 'plan-eng-review/**', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-conductor-prose.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/helpers/native-auto-decide.ts', 'test/native-auto-decide.test.ts', 'test/native-auto-decide-pty.test.ts', 'test/fixtures/native-auto-decide-ag.json', 'test/eng-seeded-completion-ai.test.ts', 'test/fixtures/eng-seeded-completion-ai.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**'], // Real-PTY E2E batch (#6 new tests on the harness). @@ -188,7 +194,7 @@ export const E2E_TOUCHFILES: Record = { // the selector sets GSTACK_CARVE_SKILL= to scope cost to the changed // skill (D-CODEX A). Touching the registry/helper or sections.ts runs all. 'carve-section-loading': ['scripts/resolvers/composition.ts', 'test/autoplan-review-discovery.test.ts', 'test/autoplan-phase-order.test.ts', 'design-html/**', 'design-shotgun/**', 'qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/resolvers/redact-doc.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/session-runner-tools.test.ts', 'test/carve-section-loading.test.ts', 'bin/gstack-autoplan-snapshot.ts', 'test/autoplan-snapshot.test.ts', 'test/autoplan-init.test.ts', 'test/autoplan-obligations.test.ts', 'test/fixtures/autoplan/t-ceo-omitted-obligations.json', 'test/fixtures/autoplan/u-ceo-original-loss.json', 'test/fixtures/autoplan/v-ceo-dangling-references.json'], - 'autoplan-chain-pty': ['scripts/resolvers/composition.ts', 'test/autoplan-review-discovery.test.ts', 'test/autoplan-phase-order.test.ts', 'autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/autoplan-dashboard.md', 'test/autoplan-chain-fixture.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'bin/gstack-config', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/autoplan-setup-question.ts', 'test/autoplan-setup-question.test.ts', 'test/helpers/autoplan-phase-observer.ts', 'test/autoplan-phase-observer.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/skill-e2e-autoplan-chain.test.ts', 'bin/gstack-autoplan-snapshot.ts', 'test/autoplan-snapshot.test.ts', 'test/autoplan-init.test.ts', 'test/autoplan-obligations.test.ts', 'test/fixtures/autoplan/t-ceo-omitted-obligations.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/helpers/eval-budgets.ts', 'test/autoplan-eval-budget.test.ts', 'test/eval-budgets-policy.test.ts', 'test/eval-detach-timeout-floor.test.ts', 'scripts/test-paid-shards.ts', '.github/workflows/evals-periodic.yml', 'test/fixtures/autoplan-routing-n-screen.txt', 'test/autoplan-routing-o.test.ts', 'test/fixtures/autoplan-routing-o-screen.txt', 'test/autoplan-setup-packet-o.test.ts', 'test/fixtures/autoplan-setup-packet-o-screen.txt', 'test/fixtures/autoplan-setup-packet-o-call.json', 'test/fixtures/autoplan/u-ceo-original-loss.json', 'test/fixtures/autoplan/v-ceo-dangling-references.json', 'test/fixtures/autoplan-setup-z-packet.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/helpers/autoplan-method-read-audit.ts', 'test/autoplan-method-read-audit.test.ts', 'test/fixtures/autoplan-method-read-aa-events.json', 'test/autoplan-routing-manual-skills.test.ts', 'test/fixtures/autoplan-routing-manual-skills-ac.json', 'test/helpers/plan-count-pending-question.ts', 'test/autoplan-pending-question.test.ts', 'test/plan-pending-question-pty.test.ts', 'test/pending-question-completion.test.ts', 'test/fixtures/pending-question-completion-ad.json', 'test/fixtures/autoplan-setup-ad-v2-packet.json', 'test/helpers/autoplan-artifact-permission.ts', 'test/autoplan-artifact-permission.test.ts', 'test/fixtures/autoplan-artifact-permission-ad-v3.json', 'scripts/resolvers/testing.ts', 'test/helpers/autoplan-artifact-recorder.ts', 'test/autoplan-artifact-recorder.test.ts', 'test/autoplan-pending-artifact.test.ts', 'test/fixtures/autoplan-pending-artifact-ae.json', 'test/autoplan-edit-header-ag.test.ts', 'test/fixtures/autoplan-edit-header-ag.json', 'test/autoplan-edit-prefix-ai.test.ts', 'test/fixtures/autoplan-edit-prefix-ai.json', 'test/autoplan-edit-panel-aj.test.ts', 'test/fixtures/autoplan-edit-panel-aj.json', + 'autoplan-chain-pty': ['scripts/resolvers/composition.ts', 'test/autoplan-review-discovery.test.ts', 'test/autoplan-phase-order.test.ts', 'autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/autoplan-dashboard.md', 'test/autoplan-chain-fixture.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'bin/gstack-config', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/autoplan-setup-question.ts', 'test/autoplan-setup-question.test.ts', 'test/helpers/autoplan-phase-observer.ts', 'test/autoplan-phase-observer.test.ts', 'test/autoplan-phase-dash-ao.test.ts', 'test/fixtures/autoplan-phase-dash-ao.json', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/skill-e2e-autoplan-chain.test.ts', 'bin/gstack-autoplan-snapshot.ts', 'test/autoplan-snapshot.test.ts', 'test/autoplan-init.test.ts', 'test/autoplan-obligations.test.ts', 'test/fixtures/autoplan/t-ceo-omitted-obligations.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/helpers/eval-budgets.ts', 'test/autoplan-eval-budget.test.ts', 'test/eval-budgets-policy.test.ts', 'test/eval-detach-timeout-floor.test.ts', 'scripts/test-paid-shards.ts', '.github/workflows/evals-periodic.yml', 'test/fixtures/autoplan-routing-n-screen.txt', 'test/autoplan-routing-o.test.ts', 'test/fixtures/autoplan-routing-o-screen.txt', 'test/autoplan-setup-packet-o.test.ts', 'test/fixtures/autoplan-setup-packet-o-screen.txt', 'test/fixtures/autoplan-setup-packet-o-call.json', 'test/fixtures/autoplan/u-ceo-original-loss.json', 'test/fixtures/autoplan/v-ceo-dangling-references.json', 'test/fixtures/autoplan-setup-z-packet.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/helpers/autoplan-method-read-audit.ts', 'test/autoplan-method-read-audit.test.ts', 'test/fixtures/autoplan-method-read-aa-events.json', 'test/autoplan-routing-manual-skills.test.ts', 'test/fixtures/autoplan-routing-manual-skills-ac.json', 'test/helpers/plan-count-pending-question.ts', 'test/autoplan-pending-question.test.ts', 'test/plan-pending-question-pty.test.ts', 'test/pending-question-completion.test.ts', 'test/fixtures/pending-question-completion-ad.json', 'test/fixtures/autoplan-setup-ad-v2-packet.json', 'test/helpers/autoplan-artifact-permission.ts', 'test/autoplan-artifact-permission.test.ts', 'test/fixtures/autoplan-artifact-permission-ad-v3.json', 'scripts/resolvers/testing.ts', 'test/helpers/autoplan-artifact-recorder.ts', 'test/autoplan-artifact-recorder.test.ts', 'test/autoplan-pending-artifact.test.ts', 'test/fixtures/autoplan-pending-artifact-ae.json', 'test/autoplan-edit-header-ag.test.ts', 'test/fixtures/autoplan-edit-header-ag.json', 'test/autoplan-edit-prefix-ai.test.ts', 'test/fixtures/autoplan-edit-prefix-ai.json', 'test/autoplan-edit-panel-aj.test.ts', 'test/fixtures/autoplan-edit-panel-aj.json', 'test/autoplan-repeated-header-ak.test.ts', 'test/fixtures/autoplan-repeated-header-ak.json', "test/helpers/autoplan-artifact-digest.ts", @@ -198,13 +204,15 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/autoplan-edit-queue-am.json", "test/autoplan-edit-edges-an.test.ts", "test/fixtures/autoplan-edit-edges-an.json", + 'test/autoplan-final-gate-ao.test.ts', + 'test/fixtures/autoplan-final-gate-ao.json', ], // Per-finding AskUserQuestion count + review-report-at-bottom assertion. // Each test drives its skill end-to-end; touchfiles include preamble + // completion-status resolvers because they affect question cadence and // terminal output (the regression surface this test catches). - 'plan-ceo-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'test/skill-ceo-section-ordering.test.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/ceo-completion-handoff.ts', 'test/ceo-completion-handoff.test.ts', 'test/ceo-count-s-terminals.test.ts', 'test/fixtures/ceo-count-s-paired.json', 'test/fixtures/ceo-completion-handoff-calls.json', 'test/fixtures/ceo-completion-handoff-j-calls.json', 'test/fixtures/ceo-completion-handoff-k-calls.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-ceo-finding-count.test.ts', 'test/ceo-completion-handoff-l.test.ts', 'test/fixtures/ceo-completion-handoff-l-calls.json', 'test/ceo-completion-handoff-m.test.ts', 'test/fixtures/ceo-completion-handoff-m-call.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/ceo-handoff-n-calls.json', 'test/ceo-completion-handoff-o.test.ts', 'test/fixtures/ceo-completion-handoff-o-call.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/helpers/ceo-approach-pick.ts', 'test/ceo-approach-pick.test.ts', 'test/fixtures/ceo-approach-q-call.json', 'test/fixtures/ceo-approach-r-call.json', 'test/fixtures/ceo-approach-r-distinct-call.json', 'test/fixtures/ceo-approach-q-paired-call.json', 'test/fixtures/ceo-completion-handoff-q-call.json', 'test/fixtures/ceo-completion-handoff-r-calls.json', 'test/fixtures/ceo-completion-handoff-t-call.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/ceo-completion-handoff-u-call.json', 'test/fixtures/ceo-completion-handoff-v-call.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/plan-count-ceo-body-finding.test.ts', 'test/fixtures/ceo-count-w-paired.json', 'test/fixtures/ceo-completion-handoff-w-call.json', 'test/fixtures/ceo-approach-y-call.json', 'test/fixtures/ceo-approach-y-screen.txt', 'test/ceo-handoff-y.test.ts', 'test/fixtures/ceo-handoff-y-call.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/ceo-handoff-z-call.json', 'test/fixtures/ceo-approach-aa-call.json', 'test/review-handoffs-aa.test.ts', 'test/fixtures/review-handoff-aa-ceo.json', 'test/fixtures/review-handoff-aa-dx.json', 'test/helpers/ceo-mode-option.ts', 'test/ceo-count-mode.test.ts', 'test/fixtures/ceo-count-mode-ab-call.json', 'test/ceo-count-ac.test.ts', 'test/fixtures/ceo-count-ac-calls.json', 'test/fixtures/ceo-count-ac-later-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/fixtures/plan-count-permission-ae.json', 'test/ceo-count-ad-v2.test.ts', 'test/fixtures/ceo-count-ad-v2.json', 'test/fixtures/plan-count-permission-target-ad-v2.json', 'test/fixtures/ceo-finding-alias-af.json', 'test/fixtures/ceo-numbered-brief-af.json', 'test/ceo-contract-assertions-ag.test.ts', 'test/fixtures/ceo-contract-assertions-ag.json', 'test/fixtures/ceo-contract-assertions-ag-retry.json', 'test/fixtures/plan-count-permission-ah.json', 'test/ceo-parenthesized-issue-ah.test.ts', 'test/fixtures/ceo-parenthesized-issue-ah.json', 'test/ceo-section-choice-ai.test.ts', 'test/fixtures/ceo-section-choice-ai.json', 'test/ceo-annotation-aj.test.ts', 'test/fixtures/ceo-annotation-aj.json', + 'plan-ceo-finding-count': ['test/dx-manual-handoff-ao.test.ts', 'test/fixtures/dx-manual-handoff-ao.json', 'bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'test/skill-ceo-section-ordering.test.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/ceo-completion-handoff.ts', 'test/ceo-completion-handoff.test.ts', 'test/ceo-count-s-terminals.test.ts', 'test/fixtures/ceo-count-s-paired.json', 'test/fixtures/ceo-completion-handoff-calls.json', 'test/fixtures/ceo-completion-handoff-j-calls.json', 'test/fixtures/ceo-completion-handoff-k-calls.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-ceo-finding-count.test.ts', 'test/ceo-completion-handoff-l.test.ts', 'test/fixtures/ceo-completion-handoff-l-calls.json', 'test/ceo-completion-handoff-m.test.ts', 'test/fixtures/ceo-completion-handoff-m-call.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/ceo-handoff-n-calls.json', 'test/ceo-completion-handoff-o.test.ts', 'test/fixtures/ceo-completion-handoff-o-call.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/helpers/ceo-approach-pick.ts', 'test/ceo-approach-pick.test.ts', 'test/fixtures/ceo-approach-q-call.json', 'test/fixtures/ceo-approach-r-call.json', 'test/fixtures/ceo-approach-r-distinct-call.json', 'test/fixtures/ceo-approach-q-paired-call.json', 'test/fixtures/ceo-completion-handoff-q-call.json', 'test/fixtures/ceo-completion-handoff-r-calls.json', 'test/fixtures/ceo-completion-handoff-t-call.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/ceo-completion-handoff-u-call.json', 'test/fixtures/ceo-completion-handoff-v-call.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/plan-count-ceo-body-finding.test.ts', 'test/fixtures/ceo-count-w-paired.json', 'test/fixtures/ceo-completion-handoff-w-call.json', 'test/fixtures/ceo-approach-y-call.json', 'test/fixtures/ceo-approach-y-screen.txt', 'test/ceo-handoff-y.test.ts', 'test/fixtures/ceo-handoff-y-call.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/ceo-handoff-z-call.json', 'test/fixtures/ceo-approach-aa-call.json', 'test/review-handoffs-aa.test.ts', 'test/fixtures/review-handoff-aa-ceo.json', 'test/fixtures/review-handoff-aa-dx.json', 'test/helpers/ceo-mode-option.ts', 'test/ceo-count-mode.test.ts', 'test/fixtures/ceo-count-mode-ab-call.json', 'test/ceo-count-ac.test.ts', 'test/fixtures/ceo-count-ac-calls.json', 'test/fixtures/ceo-count-ac-later-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/fixtures/plan-count-permission-ae.json', 'test/ceo-count-ad-v2.test.ts', 'test/fixtures/ceo-count-ad-v2.json', 'test/fixtures/plan-count-permission-target-ad-v2.json', 'test/fixtures/ceo-finding-alias-af.json', 'test/fixtures/ceo-numbered-brief-af.json', 'test/ceo-contract-assertions-ag.test.ts', 'test/fixtures/ceo-contract-assertions-ag.json', 'test/fixtures/ceo-contract-assertions-ag-retry.json', 'test/fixtures/plan-count-permission-ah.json', 'test/ceo-parenthesized-issue-ah.test.ts', 'test/fixtures/ceo-parenthesized-issue-ah.json', 'test/ceo-section-choice-ai.test.ts', 'test/fixtures/ceo-section-choice-ai.json', 'test/ceo-annotation-aj.test.ts', 'test/fixtures/ceo-annotation-aj.json', 'test/plan-count-crop-ak.test.ts', 'test/fixtures/plan-count-crop-ak.json', 'test/ceo-numbered-brief-ak.test.ts', @@ -221,8 +229,10 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/ceo-contract-question-an.json", "test/fixtures/ceo-section-finding-an.json", "test/fixtures/ceo-current-contract-an.json", + 'test/ceo-test-subject-ao.test.ts', + 'test/fixtures/ceo-test-subject-ao.json', ], - 'plan-eng-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-eng-finding-count.test.ts', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/eng-first-review-t.test.ts', 'test/fixtures/eng-batching-t-calls.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/eng-scope-y.test.ts', 'test/fixtures/eng-scope-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/eng-binding-z.test.ts', 'test/fixtures/eng-binding-z-calls.json', 'test/eng-binding-retry-z.test.ts', 'test/fixtures/eng-binding-retry-z-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/fixtures/plan-count-permission-ae.json', 'test/fixtures/plan-count-permission-target-ad-v2.json', 'test/helpers/eng-completion-handoff.ts', 'test/eng-count-ad-v2.test.ts', 'test/fixtures/eng-count-ad-v2.json', 'test/helpers/eng-seeded-coverage.ts', 'test/eng-seeded-coverage.test.ts', 'test/eng-seeded-packet-ae.test.ts', 'test/fixtures/eng-seeded-packet-ae.json', 'test/eng-first-category-af.test.ts', 'test/fixtures/eng-first-category-af.json', 'test/eng-regression-pinning-ag.test.ts', 'test/fixtures/eng-regression-pinning-ag.json', 'test/fixtures/plan-count-permission-ah.json', 'test/eng-next-handoff-ah.test.ts', 'test/fixtures/eng-next-handoff-ah.json', 'test/eng-declared-regression-ai.test.ts', 'test/fixtures/eng-declared-regression-ai.json', 'test/eng-snapshot-adapter-aj.test.ts', 'test/fixtures/eng-snapshot-adapter-aj.json', + 'plan-eng-finding-count': ['test/dx-manual-handoff-ao.test.ts', 'test/fixtures/dx-manual-handoff-ao.json', 'bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-eng-finding-count.test.ts', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/eng-first-review-t.test.ts', 'test/fixtures/eng-batching-t-calls.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/eng-scope-y.test.ts', 'test/fixtures/eng-scope-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/eng-binding-z.test.ts', 'test/fixtures/eng-binding-z-calls.json', 'test/eng-binding-retry-z.test.ts', 'test/fixtures/eng-binding-retry-z-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/fixtures/plan-count-permission-ae.json', 'test/fixtures/plan-count-permission-target-ad-v2.json', 'test/helpers/eng-completion-handoff.ts', 'test/eng-count-ad-v2.test.ts', 'test/fixtures/eng-count-ad-v2.json', 'test/helpers/eng-seeded-coverage.ts', 'test/eng-seeded-coverage.test.ts', 'test/eng-seeded-packet-ae.test.ts', 'test/fixtures/eng-seeded-packet-ae.json', 'test/eng-first-category-af.test.ts', 'test/fixtures/eng-first-category-af.json', 'test/eng-regression-pinning-ag.test.ts', 'test/fixtures/eng-regression-pinning-ag.json', 'test/fixtures/plan-count-permission-ah.json', 'test/eng-next-handoff-ah.test.ts', 'test/fixtures/eng-next-handoff-ah.json', 'test/eng-declared-regression-ai.test.ts', 'test/fixtures/eng-declared-regression-ai.json', 'test/eng-snapshot-adapter-aj.test.ts', 'test/fixtures/eng-snapshot-adapter-aj.json', 'test/eng-declared-suite-ak.test.ts', 'test/fixtures/eng-declared-suite-ak.json', 'test/plan-count-crop-ak.test.ts', @@ -242,7 +252,7 @@ export const E2E_TOUCHFILES: Record = { "test/eng-golden-parity-an.test.ts", "test/fixtures/eng-golden-parity-an.json", ], - 'plan-design-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/design-count-review.ts', 'test/design-count-review.test.ts', 'test/fixtures/design-review-j-calls.json', 'test/helpers/design-count-outside.ts', 'test/design-count-outside.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts', 'test/fixtures/design-review-l-calls.json', 'test/design-completion-handoff.test.ts', 'test/fixtures/design-handoff-l-calls.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/design-review-n-calls.json', 'test/design-completion-handoff-scored.test.ts', 'test/fixtures/design-handoff-n-calls.json', 'test/fixtures/design-handoff-q-calls.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/design-completion-handoff-u.test.ts', 'test/fixtures/design-handoff-u-calls.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/helpers/design-artifact-question.ts', 'test/design-artifact-question.test.ts', 'test/fixtures/design-artifacts-w-calls.json', 'test/fixtures/design-outside-y-calls.json', 'test/fixtures/design-boundaries-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/design-gap-z-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/fixtures/plan-count-permission-ae.json', 'test/fixtures/plan-count-permission-target-ad-v2.json', 'test/design-count-ad-v2.test.ts', 'test/fixtures/design-count-ad-v2.json', 'test/design-first-decision-af.test.ts', 'test/fixtures/design-first-decision-af.json', 'test/fixtures/design-first-decision-af-retry.json', 'test/fixtures/plan-count-permission-ah.json', 'test/design-first-issue-ai.test.ts', 'test/fixtures/design-first-issue-ai.json', 'test/design-primary-action-aj.test.ts', 'test/fixtures/design-primary-action-aj.json', 'test/fixtures/design-future-todo-aj.json', + 'plan-design-finding-count': ['test/dx-manual-handoff-ao.test.ts', 'test/fixtures/dx-manual-handoff-ao.json', 'bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/design-count-review.ts', 'test/design-count-review.test.ts', 'test/fixtures/design-review-j-calls.json', 'test/helpers/design-count-outside.ts', 'test/design-count-outside.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts', 'test/fixtures/design-review-l-calls.json', 'test/design-completion-handoff.test.ts', 'test/fixtures/design-handoff-l-calls.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/design-review-n-calls.json', 'test/design-completion-handoff-scored.test.ts', 'test/fixtures/design-handoff-n-calls.json', 'test/fixtures/design-handoff-q-calls.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/design-completion-handoff-u.test.ts', 'test/fixtures/design-handoff-u-calls.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/helpers/design-artifact-question.ts', 'test/design-artifact-question.test.ts', 'test/fixtures/design-artifacts-w-calls.json', 'test/fixtures/design-outside-y-calls.json', 'test/fixtures/design-boundaries-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/design-gap-z-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/fixtures/plan-count-permission-ae.json', 'test/fixtures/plan-count-permission-target-ad-v2.json', 'test/design-count-ad-v2.test.ts', 'test/fixtures/design-count-ad-v2.json', 'test/design-first-decision-af.test.ts', 'test/fixtures/design-first-decision-af.json', 'test/fixtures/design-first-decision-af-retry.json', 'test/fixtures/plan-count-permission-ah.json', 'test/design-first-issue-ai.test.ts', 'test/fixtures/design-first-issue-ai.json', 'test/design-primary-action-aj.test.ts', 'test/fixtures/design-primary-action-aj.json', 'test/fixtures/design-future-todo-aj.json', 'test/design-primary-contract-ak.test.ts', 'test/fixtures/design-primary-contract-ak.json', 'test/plan-count-crop-ak.test.ts', @@ -256,8 +266,12 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/design-variant-choice-am-retry.json", "test/design-primary-composition-an.test.ts", "test/fixtures/design-primary-composition-an.json", + "test/design-primary-treatment-ao.test.ts", + "test/fixtures/design-primary-treatment-ao.json", + "test/design-primary-assignment-ao.test.ts", + "test/fixtures/design-primary-assignment-ao.json", ], - 'plan-devex-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/devex-count-fixture.ts', 'test/devex-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts', 'test/fixtures/dx-prerequisite-r-call.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/devex-review-n-calls.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/devex-output-o.test.ts', 'test/fixtures/devex-review-o-calls.json', 'test/fixtures/devex-output-o-retry-call.json', 'test/devex-setup-remedy-o.test.ts', 'test/fixtures/devex-review-o-retry-calls.json', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/fixtures/devex-review-t-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/devex-count-u-calls.json', 'test/fixtures/devex-count-u-retry-calls.json', 'test/fixtures/devex-empathy-v-calls.json', 'test/fixtures/devex-handoff-v-call.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/fixtures/devex-count-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/devex-count-z-calls.json', 'test/fixtures/devex-handoff-z-call.json', 'test/review-handoffs-aa.test.ts', 'test/fixtures/review-handoff-aa-ceo.json', 'test/fixtures/review-handoff-aa-dx.json', 'test/devex-empathy-ab.test.ts', 'test/fixtures/devex-empathy-ab-calls.json', 'test/devex-ac-accounting.test.ts', 'test/fixtures/devex-ac-first-attempt-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/fixtures/plan-count-permission-ae.json', 'test/fixtures/plan-count-permission-target-ad-v2.json', 'test/devex-reconfirmation-ad-v2.test.ts', 'test/fixtures/devex-reconfirmation-ad-v2.json', 'test/plan-count-history.test.ts', 'test/helpers/devex-seed-coverage.ts', 'test/devex-seed-coverage.test.ts', 'test/fixtures/devex-seed-coverage-ad-v3.json', 'test/fixtures/plan-count-permission-ah.json', + 'plan-devex-finding-count': ['test/dx-manual-handoff-ao.test.ts', 'test/fixtures/dx-manual-handoff-ao.json', 'bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/devex-count-fixture.ts', 'test/devex-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts', 'test/fixtures/dx-prerequisite-r-call.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/devex-review-n-calls.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/devex-output-o.test.ts', 'test/fixtures/devex-review-o-calls.json', 'test/fixtures/devex-output-o-retry-call.json', 'test/devex-setup-remedy-o.test.ts', 'test/fixtures/devex-review-o-retry-calls.json', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/fixtures/devex-review-t-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/devex-count-u-calls.json', 'test/fixtures/devex-count-u-retry-calls.json', 'test/fixtures/devex-empathy-v-calls.json', 'test/fixtures/devex-handoff-v-call.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/fixtures/devex-count-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/devex-count-z-calls.json', 'test/fixtures/devex-handoff-z-call.json', 'test/review-handoffs-aa.test.ts', 'test/fixtures/review-handoff-aa-ceo.json', 'test/fixtures/review-handoff-aa-dx.json', 'test/devex-empathy-ab.test.ts', 'test/fixtures/devex-empathy-ab-calls.json', 'test/devex-ac-accounting.test.ts', 'test/fixtures/devex-ac-first-attempt-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/fixtures/plan-count-permission-ae.json', 'test/fixtures/plan-count-permission-target-ad-v2.json', 'test/devex-reconfirmation-ad-v2.test.ts', 'test/fixtures/devex-reconfirmation-ad-v2.json', 'test/plan-count-history.test.ts', 'test/helpers/devex-seed-coverage.ts', 'test/devex-seed-coverage.test.ts', 'test/fixtures/devex-seed-coverage-ad-v3.json', 'test/fixtures/plan-count-permission-ah.json', 'test/dx-signature-identity-ak.test.ts', 'test/fixtures/dx-signature-identity-ak.json', 'test/plan-count-crop-ak.test.ts', diff --git a/test/skill-e2e-autoplan-chain.test.ts b/test/skill-e2e-autoplan-chain.test.ts index efa4303a3..c958cb4a4 100644 --- a/test/skill-e2e-autoplan-chain.test.ts +++ b/test/skill-e2e-autoplan-chain.test.ts @@ -32,7 +32,7 @@ import { } from './helpers/claude-pty-runner'; import { autoplanArtifactPermissionInput, pendingAutoplanArtifactPermissionInput, publishedAutoplanArtifactPermissionInput, autoplanArtifactMenuKey } from './helpers/autoplan-artifact-permission'; import { readPendingAutoplanArtifact, autoplanArtifactRecorderStatus } from './helpers/autoplan-artifact-recorder'; -import { autoplanSetupDecision, type AutoplanSetupDecision } from './helpers/autoplan-setup-question'; +import { autoplanSetupDecision, autoplanBlockingQuestionBoundary, type AutoplanSetupDecision } from './helpers/autoplan-setup-question'; import { autoplanPhaseCompletions, type AutoplanPhaseHit } from './helpers/autoplan-phase-observer'; import { readPlanCountTranscript, type PlanCountTranscript, type NativePublicToolEvent } from './helpers/plan-count-transcript'; import { readPendingQuestion, pendingQuestionRecorderStatus } from './helpers/plan-count-pending-question'; @@ -92,8 +92,9 @@ describeE2E('/autoplan native chain ordering (periodic)', () => { let pendingArtifact: ReturnType; let viewportCapturedAt = Date.now(); let methodologyAudit: AutoplanMethodReadAudit[] = []; - let outcome: 'chain_complete' | 'plan_ready' | 'timeout' | 'exited' | 'unsupported_setup' | 'incomplete_methodology' = 'timeout'; + let outcome: 'chain_complete' | 'plan_ready' | 'timeout' | 'exited' | 'unsupported_setup' | 'incomplete_methodology' | 'blocked_on_question' = 'timeout'; let unsupportedSetup: Extract | null = null; + let blockedQuestion: ReturnType = null; let evidence = ''; let viewport = ''; let fullSessionEvidence = ''; @@ -119,7 +120,7 @@ describeE2E('/autoplan native chain ordering (periodic)', () => { artifacts = saveSnapshot({ skillName: 'autoplan', cwd: tempDir, claudeConfigDir: session.hermeticConfigDir, raw: session.rawOutput(), visible: session.visibleText(), viewport, - observation: { state, hits, native: transcript, pendingSetupQuestion, pendingArtifact, methodologyAudit, exitCode: session.exitCode(), unsupportedSetup, + observation: { state, hits, native: transcript, pendingSetupQuestion, pendingArtifact, methodologyAudit, exitCode: session.exitCode(), unsupportedSetup, blockedQuestion, ownedArtifactStateRoot: session.hermeticSkillStateRoot, artifactRecorder:autoplanArtifactRecorderStatus(session.pendingAutoplanArtifactFile, tempDir, session.hermeticConfigDir, session.hermeticSkillStateRoot), pendingQuestionRecorder:pendingQuestionRecorderStatus(session.pendingQuestionFile, tempDir, session.hermeticConfigDir), @@ -226,6 +227,17 @@ describeE2E('/autoplan native chain ordering (periodic)', () => { break; } + // Autoplan auto-decides until its final human gate. An unrelated + // unanswered question blocks this test; never supply an answer or credit. + // Keep waiting on already-handled/partial setup panels as before. + blockedQuestion = setup.kind === 'unrelated' ? autoplanBlockingQuestionBoundary(visible, { + commandStartedAt, viewportCapturedAt, transcript, publicTools, pending:pendingSetupQuestion}) : null; + if (blockedQuestion) { + outcome = 'blocked_on_question'; + evidence = visible.slice(-3000); + break; + } + // Plan-ready as a fallback terminal — autoplan finished without // surfacing a Phase 3 marker. This is a regression surface. if (isPlanReadyVisible(visible)) { @@ -252,6 +264,16 @@ describeE2E('/autoplan native chain ordering (periodic)', () => { } finally { await session.close(); } } + if (outcome === 'blocked_on_question') { + const missing = [1, 2, 2.5, 3].filter(phase => !hits.some(hit => hit.phase === phase)); + throw new Error( + `autoplan chain test FAILED: outcome=blocked_on_question; missing phase markers=${JSON.stringify(missing)}; ` + + `question=${JSON.stringify(blockedQuestion)}; no input sent.\n` + + `Native transcript: ${transcript.status}; artifacts=${JSON.stringify(artifacts)}\n` + + `--- evidence ---\n${evidence}`, + ); + } + if (outcome === 'exited' || outcome === 'timeout' || outcome === 'unsupported_setup' || outcome === 'incomplete_methodology') { throw new Error( `autoplan chain test FAILED: outcome=${outcome}, exitCode=${exitCode}, hits=${JSON.stringify(hits)}\n` + diff --git a/test/skill-e2e-conductor-prose.test.ts b/test/skill-e2e-conductor-prose.test.ts index 7ddf04837..de2e5e9aa 100644 --- a/test/skill-e2e-conductor-prose.test.ts +++ b/test/skill-e2e-conductor-prose.test.ts @@ -47,6 +47,9 @@ describeE2E('Conductor renders decisions as prose (periodic)', () => { extraArgs: ['--disallowedTools', 'AskUserQuestion'], env: { CONDUCTOR_WORKSPACE_PATH: '/tmp/conductor-prose-e2e' }, initialPlanContent: FLAWED_PLAN, + // A judge can mistake a streaming partial option for a waiting brief. + // Keep observing until the independent prose evidence is available. + requireProseEvidence: true, timeoutMs: CAPTURE_MS, });