mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-06 17:04:27 +01:00
fix: count native review decisions and isolate eval fixtures
This commit is contained in:
Garry Tan committed
1 parent
7fac51e9bd
commit
4636893f52
30 files changed
+5491
-33
No files matched your search
@@ -264,3 +264,102 @@ fs.writeFileSync(${JSON.stringify(resultFile)},JSON.stringify(results));
|
||||
}fs.rmSync(dir,{recursive:true,force:true});
|
||||
}
|
||||
},30000);
|
||||
|
||||
const adV2Packet = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/autoplan-setup-ad-v2-packet.json'), 'utf8')) as {pendingCall: NativePlanQuestionCall; screen: string};
|
||||
test('AD v2 actual setup packet chooses routing and standard review with the existing native identity',()=>{
|
||||
const seen=new Set<string>(),call=adV2Packet.pendingCall;
|
||||
expect(autoplanSetupDecision(adV2Packet.screen,seen).kind).toBe('waiting');
|
||||
commit(adV2Packet.screen,seen,call,'1');
|
||||
// Only the first pane was retained live. Later panes are explicit native-question projections.
|
||||
commit(pane(call,1,[0]),seen,call,'2');
|
||||
commit(pane(call,2,[0,1]),seen,call,'\r');
|
||||
});
|
||||
|
||||
test('AD v2 setup policy uses the task and actions across presentation and option order',()=>{
|
||||
for(const variant of ['numbered','unprefixed','different explanation'])for(const reverseQuestions of [false,true])for(const reverseOptions of [false,true]){
|
||||
const call=structuredClone(adV2Packet.pendingCall);
|
||||
call.questions.forEach((q,index)=>{
|
||||
q.question=q.question.replace(/^D\d+\s*[—–:-]\s*/,variant==='unprefixed'?'':`D${31+index}: `);
|
||||
if(variant==='different explanation')q.question=q.question.split('\n')[0]+'\nProject/branch/task: disposable review fixture, another branch and release.\nELI10: This setup changes how later sessions find workflow context.\nStakes if we pick wrong: an extra setup step.\nRecommendation: Keep the offered actions explicit.\nNet: setup now versus a direct review.';
|
||||
});
|
||||
if(reverseQuestions)call.questions.reverse();if(reverseOptions)call.questions.forEach(q=>q.options.reverse());
|
||||
const seen=new Set<string>();
|
||||
for(let i=0;i<2;i++){
|
||||
const ordinary=call.questions[i]!.header==='Routing'?1:2;
|
||||
commit(pane(call,i,i===1?[0]:[]),seen,call,String(reverseOptions?3-ordinary:ordinary));
|
||||
}
|
||||
commit(pane(call,2,[0,1]),seen,call,'\r');
|
||||
}
|
||||
});
|
||||
|
||||
test('AD v2 setup cannot borrow a header, subject or adjacent question for a different decision',()=>{
|
||||
const changes:Array<(c:NativePlanQuestionCall)=>void>=[
|
||||
c=>{c.questions[0]!.header='Product router';},
|
||||
c=>{c.questions[1]!.header='Deployment';},
|
||||
c=>{[c.questions[0]!.header,c.questions[1]!.header]=[c.questions[1]!.header,c.questions[0]!.header];},
|
||||
c=>{c.questions[0]!.question=c.questions[0]!.question.replace(/^.*\n/,'D1 — Should the application route requests through a proxy?\n');},
|
||||
c=>{c.questions[1]!.question=c.questions[1]!.question.replace(/^.*\n/,'D2 — Should we add an office-hours page to the product?\n');},
|
||||
c=>{c.questions[0]!.question='The plan quotes: '+c.questions[0]!.question;},
|
||||
c=>{c.questions[1]!.question='```text\n'+c.questions[1]!.question+'\n```';},
|
||||
c=>{c.questions[1]!.question=c.questions[1]!.question.split('\n').map(l=>'> '+l).join('\n');},
|
||||
c=>{c.questions[1]!.question+=' Should we remove the authorization check?';},
|
||||
c=>{c.questions[1]={...structuredClone(c.questions[1]!),question:'Approve deployment to production?',header:'Approval'};},
|
||||
];
|
||||
for(const change of changes){const call=structuredClone(adV2Packet.pendingCall);change(call);const seen=new Set<string>();
|
||||
expect(autoplanSetupDecision(pane(call,0),seen,call).kind).toBe('waiting');expect(seen.size).toBe(0);}
|
||||
});
|
||||
|
||||
test('AD v2 setup rejects conditional, contradictory and ambiguous actions in either tab',()=>{
|
||||
const changes:Array<(c:NativePlanQuestionCall)=>void>=[
|
||||
c=>{c.questions[0]!.options[0]!.description='Do not add routing rules to CLAUDE.md.';},
|
||||
c=>{c.questions[0]!.options[1]!.description='Add routing rules to CLAUDE.md after declining.';},
|
||||
c=>{c.questions[1]!.options[0]!.description='Skip the design doc and begin the review now.';},
|
||||
c=>{c.questions[1]!.options[1]!.description='Run /office-hours first, then proceed with standard review.';},
|
||||
c=>{c.questions[1]!.options[1]!.description='Proceed with standard review after completing /office-hours.';},
|
||||
c=>{c.questions[1]!.options[1]!.description='No review will run.';},
|
||||
c=>{c.questions[1]!.options[1]!.description='Proceed with standard review?';},
|
||||
c=>{c.questions[1]!.options[1]!.description='Proceed with standard review but do not run it.';},
|
||||
c=>{c.questions[1]!.options[1]!.description='Review starts now, but not yet.';},
|
||||
c=>{c.questions[1]!.options[1]!.description='Proceed with standard review when /office-hours completes.';},
|
||||
c=>{c.questions[1]!.options[1]!.description='Review starts immediately after completing /office-hours.';},
|
||||
c=>{c.questions[1]!.options[1]!.description='Proceed with standard review once the design doc is complete.';},
|
||||
c=>{c.questions[1]!.options[1]!.description='Proceed with standard review if the tests pass.';},
|
||||
c=>{c.questions[1]!.options[1]!.description='Skip the CEO review and proceed directly to engineering.';},
|
||||
c=>{c.questions[1]!.options[1]!.description='Proceed with standard review only if the tests pass.';},
|
||||
c=>{c.questions[1]!.question+=' You must run /office-hours before the review.';},
|
||||
c=>{c.questions[1]!.question+=' Standard review is forbidden until /office-hours completes.';},
|
||||
c=>{c.questions[0]!.options[0]!.label+=' and implement the feature';},
|
||||
c=>{c.questions[1]!.options[1]!.label+=' if the tests pass';},
|
||||
c=>{c.questions[0]!.options[0]!.description+=' Also delete the authorization check.';},
|
||||
c=>{c.questions[1]!.options[1]!.description+=' Also deploy to production.';},
|
||||
c=>{c.questions[0]!.options[1]=structuredClone(c.questions[0]!.options[0]!);},
|
||||
c=>{c.questions[1]!.options.push({label:'Skip the remaining review phases'});},
|
||||
];
|
||||
for(const change of changes){const call=structuredClone(adV2Packet.pendingCall);change(call);const seen=new Set<string>();
|
||||
expect(autoplanSetupDecision(pane(call,0),seen,call).kind).toBe('waiting');expect(seen.size).toBe(0);}
|
||||
});
|
||||
|
||||
test('AD v2 setup retains complete native identity and current-pane requirements',()=>{
|
||||
const first=adV2Packet.screen,call=adV2Packet.pendingCall;
|
||||
// The example label must introduce the panel, not precede unrelated earlier transcript rows.
|
||||
for(const screen of ['Example panel:\n'+pane(call,0),'```text\n'+first,first+'\nContinuing.',
|
||||
first.replace('Design doc','Different tab'),first.replace('Esc to cancel','Esc to'),
|
||||
first.replace('Add routing rules to CLAUDE.md (recommended)','Add routing rules to OTHER.md (recommended)')]){
|
||||
expect(screen).not.toBe(first);expect(autoplanSetupDecision(screen,new Set(),call).kind).toBe('waiting');
|
||||
}
|
||||
for(const delta of [{answered:true},{failed:true},{sessionId:''},{toolUseId:''}])
|
||||
expect(autoplanSetupDecision(first,new Set(),{...call,...delta}).kind).toBe('waiting');
|
||||
});
|
||||
|
||||
test('AD v2 setup fixture selects the existing Autoplan paid case only',()=>{
|
||||
expect(E2E_TOUCHFILES['autoplan-chain-pty']).toContain('test/fixtures/autoplan-setup-ad-v2-packet.json');
|
||||
const owners=Object.entries(E2E_TOUCHFILES).filter(([,files])=>files.includes('test/fixtures/autoplan-setup-ad-v2-packet.json')).map(([name])=>name);
|
||||
expect(owners).toEqual(['autoplan-chain-pty']);
|
||||
});
|
||||
|
||||
test('AD v2 selected review action allows short affirmative descriptions with dynamic tradeoffs',()=>{
|
||||
for(const description of ['Proceed with standard review. The plan already states its goals.', 'Review begins now using the existing plan. No separate design artifact is created.', 'Start the standard review immediately with the supplied context.']){
|
||||
const call=structuredClone(adV2Packet.pendingCall);call.questions[1]!.options[1]!.description=description;
|
||||
expect(autoplanSetupDecision(pane(call,0),new Set(),call).kind).toBe('input');
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,125 @@
|
||||
import {expect,test} from 'bun:test';
|
||||
import fs from 'node:fs';import os from 'node:os';import path from 'node:path';
|
||||
import fixture from './fixtures/ceo-count-ad-v2.json';
|
||||
import {readPlanCountTranscript,type NativePlanQuestionCall} from './helpers/plan-count-transcript';
|
||||
import {ceoFirstReviewAUQ,ceoStep0Boundary,nativePlanCallFingerprint,planCountQuestionPhase} from './helpers/claude-pty-runner';
|
||||
import {isCeoCompletionHandoff,pickCeoCompletionHandoff} from './helpers/ceo-completion-handoff';
|
||||
import {selectTests, E2E_TOUCHFILES} from './helpers/touchfiles';
|
||||
const fp=(c:NativePlanQuestionCall)=>nativePlanCallFingerprint(c,0,true);
|
||||
const get=(which:'distinct'|'paired'|'pairedRetry',index:number)=>structuredClone(fixture.cases[which].calls[index]) as NativePlanQuestionCall;
|
||||
const realFindings=()=>[get('distinct',4),get('paired',4),get('paired',5),get('pairedRetry',2),get('pairedRetry',3)];
|
||||
const answer=(c:NativePlanQuestionCall)=>{c.answers={[c.questions[0]!.question]:c.questions[0]!.options[0]!.label};return c;};
|
||||
function count(calls:NativePlanQuestionCall[]){let started=false;const n={setup:0,review:0,administrative:0};for(const c of calls){const p=planCountQuestionPhase(fp(c),started,ceoStep0Boundary,ceoFirstReviewAUQ,undefined,isCeoCompletionHandoff);started=p.reviewStarted;n[p.administrative?'administrative':p.preReview?'setup':'review']++;}return n;}
|
||||
|
||||
test('exact public native requests and successful replies reconstruct the captured calls once',()=>{
|
||||
for(const which of ['distinct','paired','pairedRetry'] as const){const c=fixture.cases[which],dir=fs.mkdtempSync(path.join(os.tmpdir(),'ceo-count-public-'));try{const project=path.join(dir,'projects','owned');fs.mkdirSync(project,{recursive:true});const records=c.nativeRecords.map(r=>JSON.stringify(r)).join('\n')+'\n';fs.writeFileSync(path.join(project,c.calls[0]!.sessionId+'.jsonl'),records+records);expect(readPlanCountTranscript(dir,c.observation.capture.cwd).calls).toEqual(c.calls);for(const a of c.timeAnchors){expect(Date.parse(a.requestAt)).toBeLessThanOrEqual(Date.parse(a.replyAt));expect(Date.parse(a.replyAt)).toBeLessThanOrEqual(Date.parse(c.observation.capture.at));}}finally{fs.rmSync(dir,{recursive:true,force:true});}}
|
||||
});
|
||||
for(const [which,index] of [['distinct',4],['paired',4],['paired',5]] as const)test(`actual ${which} issue ${index} starts review from a completed native decision`,()=>expect(ceoFirstReviewAUQ(fp(get(which,index)))).toBe(true));
|
||||
test('exact snapshots keep real issue counts and separate the administrative handoff',()=>{
|
||||
expect(count(fixture.cases.distinct.calls as NativePlanQuestionCall[])).toEqual({setup:4,review:1,administrative:0});
|
||||
expect(count(fixture.cases.paired.calls as NativePlanQuestionCall[])).toEqual({setup:4,review:2,administrative:1});
|
||||
expect(fixture.cases.distinct.observation.state).toBe('in_progress');expect(fixture.cases.paired.observation.state).toBe('in_progress');
|
||||
expect(count(fixture.cases.distinct.calls as NativePlanQuestionCall[]).review).toBeLessThan(4);
|
||||
});
|
||||
test('finding numbering and matching header identity are presentation, not extra findings',()=>{
|
||||
for(const c of realFindings()){
|
||||
const q=c.questions[0]!,oldTitle=q.question.split('\n')[0]!;q.question=q.question.replace(/^D\d+\s*[—–-]\s*/,'');answer(c);expect(ceoFirstReviewAUQ(fp(c))).toBe(true);
|
||||
q.question=q.question.replace(/^(Finding|Issue)\s+[\d.]+:/,'$1 27.3:');if(/^(Finding|Issue)\s+[\d.]+$/i.test(q.header))q.header=q.header.replace(/[\d.]+/,'27.3');answer(c);expect(ceoFirstReviewAUQ(fp(c))).toBe(true);
|
||||
expect(oldTitle).toContain('?');
|
||||
}
|
||||
});
|
||||
test('native completion, identity, offered answer and unambiguous single issue remain mandatory',()=>{
|
||||
const mutations:Array<(c:NativePlanQuestionCall)=>void>=[c=>{c.answered=false;},c=>{c.failed=true;},c=>{c.answers={};},c=>{c.unansweredQuestionIndices=[0];},c=>{c.questions[0]!.multiSelect=true;},c=>{c.answers={[c.questions[0]!.question]:'unoffered'};},c=>{c.questions.push(structuredClone(c.questions[0]!));},c=>{c.questions[0]!.options[1]!.label=c.questions[0]!.options[0]!.label;answer(c);}];
|
||||
for(const mutate of mutations)for(const c of realFindings()){mutate(c);expect(ceoFirstReviewAUQ(fp(c))).toBe(false);}
|
||||
for(const c of realFindings()){expect(ceoFirstReviewAUQ({...fp(c),signature:'foreign:call'})).toBe(false);expect(ceoFirstReviewAUQ({...fp(c),nativeCall:undefined})).toBe(false);expect(ceoFirstReviewAUQ({...fp(c),options:[]})).toBe(false);}
|
||||
});
|
||||
test('setup, quoted examples, foreign qids and contradictory numbered headers cannot start review',()=>{
|
||||
for(const prefix of ['Example: ','> ','"','```\n'])for(const c of realFindings()){c.questions[0]!.question=prefix+c.questions[0]!.question;expect(ceoFirstReviewAUQ(fp(answer(c)))).toBe(false);}
|
||||
for(const header of ['Approach','Mode','Next review','Setup','Finding 88','Issue 88'])for(const c of realFindings()){c.questions[0]!.header=header;expect(ceoFirstReviewAUQ(fp(c))).toBe(false);}
|
||||
for(const c of realFindings()){c.questions[0]!.question+=' <gstack-qid:plan-eng-review-finding>';expect(ceoFirstReviewAUQ(fp(answer(c)))).toBe(false);}
|
||||
for(const which of ['distinct','paired'] as const)for(const c of fixture.cases[which].calls.slice(0,4))expect(ceoFirstReviewAUQ(fp(c as NativePlanQuestionCall))).toBe(false);
|
||||
});
|
||||
test('a completed pure next-review menu is administrative without granting pending input permission',()=>{
|
||||
const c=get('paired',6);expect(isCeoCompletionHandoff(fp(c))).toBe(true);expect(ceoFirstReviewAUQ(fp(c))).toBe(false);expect(pickCeoCompletionHandoff(fp(c))).toBeNull();c.answered=false;delete c.answers;delete c.answeredAt;c.unansweredQuestionIndices=[0];expect(isCeoCompletionHandoff(fp(c))).toBe(false);expect(pickCeoCompletionHandoff(fp(c))).toBeNull();
|
||||
});
|
||||
test('new work or uncertain closure in the next-review choice stays substantive',()=>{
|
||||
for(const suffix of ['\nFix the missing authentication check.','\nDelete the CI gate.','\nShip the new endpoint now.','\nWhich new endpoint should we add?']){const c=get('paired',6);c.questions[0]!.question+=suffix;expect(isCeoCompletionHandoff(fp(answer(c)))).toBe(false);}
|
||||
for(const change of ['CEO review is not complete.','CEO review will be complete.','Example: CEO review complete.']){const c=get('paired',6);c.questions[0]!.question=c.questions[0]!.question.replace('CEO review complete.',change);expect(isCeoCompletionHandoff(fp(answer(c)))).toBe(false);}
|
||||
for(const mutation of [c=>{c.failed=true;},c=>{c.questions[0].multiSelect=true;},c=>{c.answers={[c.questions[0].question]:'Fix the bug first'};},c=>{c.questions[0].options.push({label:'Fix the security issue',description:'Add a new check.'});}] as Array<(c:NativePlanQuestionCall)=>void>){const c=get('paired',6);mutation(c);expect(isCeoCompletionHandoff(fp(c))).toBe(false);}
|
||||
});
|
||||
|
||||
// The next-gate explanation must never turn conditional CEO closure into a
|
||||
// completed review. Its narrow normalization is for counting only.
|
||||
test('next Eng gate timing cannot supply conditional CEO completion', () => {
|
||||
for (const replacement of [
|
||||
'The CEO review is complete until someone runs it later.',
|
||||
'The CEO review is complete if someone runs it later.',
|
||||
'The CEO review will be complete after someone runs it later.',
|
||||
'The CEO review still has unresolved findings.',
|
||||
]) {
|
||||
const call = get('paired', 6);
|
||||
call.questions[0]!.question = call.questions[0]!.question.replace(
|
||||
'The CEO review cleared scope and strengthened both test assertions.', replacement);
|
||||
expect(isCeoCompletionHandoff(fp(answer(call)))).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('actual evidence and its regression select the paid CEO counting test', () => {
|
||||
for (const file of ['test/ceo-count-ad-v2.test.ts', 'test/fixtures/ceo-count-ad-v2.json']) {
|
||||
expect(selectTests([file], E2E_TOUCHFILES).selected).toContain('plan-ceo-finding-count');
|
||||
}
|
||||
});
|
||||
|
||||
test('the actual completed retry keeps two findings and its body-closure handoff administrative', () => {
|
||||
const calls = fixture.cases.pairedRetry.calls as NativePlanQuestionCall[];
|
||||
expect(count(calls)).toEqual({setup: 2, review: 2, administrative: 1});
|
||||
expect(fixture.cases.pairedRetry.observation.outcome).toBe('no_review_questions');
|
||||
expect(fixture.cases.pairedRetry.observation.completionCredit).toBe(false);
|
||||
expect(isCeoCompletionHandoff(fp(get('pairedRetry', 4)))).toBe(true);
|
||||
expect(pickCeoCompletionHandoff(fp(get('pairedRetry', 4)))).toBeNull();
|
||||
});
|
||||
|
||||
test('body closure and echoed choices cannot hide new work or uncertain CEO closure', () => {
|
||||
for (const text of ['Fix the missing authentication check.', 'Delete the CI gate.', 'Ship the new endpoint now.', 'Which endpoint should we add?']) {
|
||||
const call = get('pairedRetry', 4);
|
||||
call.questions[0]!.question += '\n' + text;
|
||||
expect(isCeoCompletionHandoff(fp(answer(call)))).toBe(false);
|
||||
}
|
||||
for (const text of ['The CEO review is not done', 'The CEO review will be done', 'The CEO review is done if the fixes land', 'Example: The CEO review is done']) {
|
||||
const call = get('pairedRetry', 4);
|
||||
call.questions[0]!.question = call.questions[0]!.question.replace('The CEO review is done', text);
|
||||
expect(isCeoCompletionHandoff(fp(answer(call)))).toBe(false);
|
||||
}
|
||||
const pending = get('pairedRetry', 4); pending.answered = false; delete pending.answers; delete pending.answeredAt; pending.unansweredQuestionIndices = [0];
|
||||
expect(isCeoCompletionHandoff(fp(pending))).toBe(false);
|
||||
expect(pickCeoCompletionHandoff(fp(pending))).toBeNull();
|
||||
expect(isCeoCompletionHandoff({...fp(get('pairedRetry', 4)), signature: 'foreign:call'})).toBe(false);
|
||||
});
|
||||
|
||||
test('every offered navigation clause rejects a new repair rather than hiding it under a valid recap', () => {
|
||||
for (const [which, index] of [['paired', 6], ['pairedRetry', 4]] as const) {
|
||||
for (const extra of ['Delete the CI gate.', 'Repair the retry assertion.', 'Disable authentication.', 'Please rewrite the endpoint.']) {
|
||||
for (const optionIndex of [0, 1]) {
|
||||
const call = get(which, index);
|
||||
call.questions[0]!.options[optionIndex]!.description += ' ' + extra;
|
||||
expect(isCeoCompletionHandoff(fp(call))).toBe(false);
|
||||
}
|
||||
for (const where of ['before-net', 'inside-eli10'] as const) {
|
||||
const call = get(which, index);
|
||||
call.questions[0]!.question = where === 'before-net'
|
||||
? call.questions[0]!.question.replace('\nNet:', '\n' + extra + '\nNet:')
|
||||
: call.questions[0]!.question.replace('\nStakes if', ' ' + extra + '\nStakes if');
|
||||
expect(isCeoCompletionHandoff(fp(answer(call)))).toBe(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('timing annotations cannot conceal substantive instructions', () => {
|
||||
for (const [which, index] of [['paired', 6], ['pairedRetry', 4]] as const) {
|
||||
for (const text of [' (human: Delete the CI gate)', ' (human: ~2 min / CC: Disable authentication)', ' (human: ~2 min / CC: ~1 min; repair the retry assertion)']) {
|
||||
const c = get(which, index); c.questions[0]!.options[0]!.description += text;
|
||||
expect(isCeoCompletionHandoff(fp(c))).toBe(false);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,75 @@
|
||||
import {expect,test} from 'bun:test';
|
||||
import {capturePlanCountQuestion,nativePlanCallFingerprint,planCountPrerequisitePick,planCountQuestionInput} from './helpers/claude-pty-runner';
|
||||
import {nextCeoModeNavigation} from './helpers/ceo-mode-option';
|
||||
import type {NativePlanQuestionCall} from './helpers/plan-count-transcript';
|
||||
import captured from './fixtures/ceo-prerequisite-ad-v2.json';
|
||||
function pending(){const c=structuredClone(captured.completedCall) as NativePlanQuestionCall;c.answered=false;delete c.answers;delete c.answeredAt;delete c.unansweredQuestionIndices;return c;}
|
||||
// Native identities and questions are exact; pending panes are synthetic projections.
|
||||
function pane(c:NativePlanQuestionCall,index:number){const q=c.questions[index]!;return [
|
||||
c.questions.length>1?'← '+c.questions.map((v,i)=>`${i<index?'☒':'☐'} ${v.header}`).join(' ')+' ✔ Submit →':'☐ '+q.header,
|
||||
q.question,...q.options.map((v,i)=>`${i?' ':'❯'} ${i+1}. ${v.label}`),
|
||||
`Enter to select · ${c.questions.length>1?'Tab/Arrow keys':'↑/↓'} to navigate · Esc to cancel`].join('\n');}
|
||||
function frame(c:NativePlanQuestionCall,index:number){const visible=pane(c,index);return {visible,active:capturePlanCountQuestion(visible,new Set(),0,true,c)!,routing:nativePlanCallFingerprint(c,0,true)};}
|
||||
test('AD v2 actual comma prerequisite selects standard review on its active native tab',()=>{
|
||||
const actual=captured.completedCall,q=actual.questions[2]!;
|
||||
expect(actual.answered).toBe(true);expect(actual.failed).toBe(false);expect(actual.answers[q.question]).toBe('Run /office-hours now');
|
||||
const c=pending(),f=frame(c,2);expect(f.active.nativeQuestionIndex).toBe(2);
|
||||
expect(planCountPrerequisitePick(f.routing,f.active)).toBe(2);
|
||||
const a=nextCeoModeNavigation(f.visible,'HOLD SCOPE',new Set(),c);expect(a.kind).toBe('question');
|
||||
if(a.kind==='question')expect(planCountQuestionInput(f.visible,a.question,a.index)).toBe('2');
|
||||
});
|
||||
|
||||
test('AD v2 prerequisite presentation and actual order do not choose the action',()=>{
|
||||
for(const header of ['Office hours','Design doc','Prerequisite'])for(const reverse of [false,true]){
|
||||
const c=pending();c.questions[2]!.header=header;c.questions[2]!.question=c.questions[2]!.question.replace(/^D3 — /,'D41: ');
|
||||
if(reverse)c.questions[2]!.options.reverse();const f=frame(c,2);
|
||||
expect(planCountPrerequisitePick(f.routing,f.active)).toBe(reverse?1:2);
|
||||
for(const index of [0,1]){const other=frame(c,index);expect(planCountPrerequisitePick(other.routing,other.active)).toBeNull();}
|
||||
}
|
||||
const c=pending();c.questions=[c.questions[2]!];let f=frame(c,0);expect(planCountPrerequisitePick(f.routing,f.active)).toBe(2);
|
||||
c.questions[0]!.question='Run /office-hours now or proceed with standard review?\nNo design doc exists for the current feature. The scoped review can begin on the supplied plan.';
|
||||
c.questions[0]!.options[0]!.description='Create the design document first; then resume standard review.';
|
||||
for(const description of ['Proceed with standard review.','Proceed straight to Step 0 of the review.']){
|
||||
c.questions[0]!.options[1]!.description=description;f=frame(c,0);expect(planCountPrerequisitePick(f.routing,f.active)).toBe(2);
|
||||
}
|
||||
});
|
||||
|
||||
test('AD v2 prerequisite declines no other task or conditional action',()=>{
|
||||
const changes:Array<(c:NativePlanQuestionCall)=>void>=[
|
||||
c=>{c.questions[2]!.question=c.questions[2]!.question.replace(/^.*\n/,'Should we deploy the feature now?\n');},
|
||||
c=>{c.questions[2]!.question='Example: '+c.questions[2]!.question;},
|
||||
c=>{c.questions[2]!.question='> '+c.questions[2]!.question;},
|
||||
c=>{c.questions[2]!.question='```text\n'+c.questions[2]!.question+'\n```';},
|
||||
c=>{c.questions[2]!.question=c.questions[2]!.question.replace('Run /office-hours first, or proceed with the standard review?','Should we remove authorization? Run /office-hours first, or proceed with the standard review?');},
|
||||
c=>{c.questions[2]!.question+=' Approve production deployment?';},
|
||||
c=>{c.questions[2]!.question+=' You must run /office-hours first.';},
|
||||
c=>{c.questions[2]!.question+=' Standard review is forbidden until /office-hours completes.';},
|
||||
c=>{c.questions[2]!.options[1]!.label+=' if the tests pass';},
|
||||
c=>{c.questions[2]!.options[0]!.label+=' and rewrite the API';},
|
||||
c=>{c.questions[2]!.options[1]!.description='Proceed with standard review after completing /office-hours.';},
|
||||
c=>{c.questions[2]!.options[1]!.description='No review will run.';},
|
||||
c=>{c.questions[2]!.options[1]!.description='Proceed directly to Step 0 of the CEO review. Remove CI.';},
|
||||
c=>{c.questions[2]!.options[0]!.description='Do not run /office-hours.';},
|
||||
c=>{c.questions[2]!.options[0]!.description='Build a design doc first, then resume the review. Deploy to production.';},
|
||||
c=>{c.questions[2]!.options[1]!.description='';},
|
||||
c=>{c.questions[2]!.options.push({label:'Approve deployment'});},
|
||||
c=>{c.questions[2]!.multiSelect=true;},
|
||||
];
|
||||
for(const change of changes){const c=pending();change(c);const f=frame(c,2);expect(planCountPrerequisitePick(f.routing,f.active)).toBeNull();}
|
||||
});
|
||||
|
||||
test('AD v2 prerequisite requires the active native packet identity',()=>{
|
||||
const c=pending(),f=frame(c,2);
|
||||
for(const active of [{...f.active,preReview:false},{...f.active,signature:'foreign:tool:question:2'},
|
||||
{...f.active,nativeQuestionIndex:0},{...f.active,promptSnippet:'Unrelated question'},
|
||||
{...f.active,nativeCall:undefined},{...f.active,options:[...f.active.options].reverse()}])
|
||||
expect(planCountPrerequisitePick(f.routing,active)).toBeNull();
|
||||
expect(planCountPrerequisitePick({...f.active,nativeCall:undefined})).toBeNull();
|
||||
for(const delta of [{answered:true},{failed:true},{sessionId:''},{toolUseId:''}]){const call={...pending(),...delta};const x=frame(call,2);expect(planCountPrerequisitePick(x.routing,x.active)).toBeNull();}
|
||||
});
|
||||
|
||||
import {E2E_TOUCHFILES,selectTests} from './helpers/touchfiles';
|
||||
test('AD v2 prerequisite regression selects its existing mode workflow',()=>{
|
||||
for(const file of ['test/ceo-prerequisite-ad-v2.test.ts','test/fixtures/ceo-prerequisite-ad-v2.json'])
|
||||
expect(selectTests([file],E2E_TOUCHFILES,[]).selected).toEqual(['plan-ceo-mode-routing']);
|
||||
});
|
||||
@@ -372,3 +372,64 @@ describe('S native same-finding ordered trace', () => {
|
||||
['explicit defect dismissal', finding + '\nThis is not a defect; no fix is required.\n'],
|
||||
])('rejects %s', (_name, text) => expect(hasStaleFillRaceFinding(text)).toBe(false));
|
||||
});
|
||||
|
||||
// Actual v2 SDK review identifies the missing fill/write coordination directly.
|
||||
// The full delivered report is retained in run evidence; this is its exact finding.
|
||||
describe('explicit uncoordinated cache-fill freshness violation', () => {
|
||||
const finding = [
|
||||
"**[Amended: D2, D3, D4, D5]** The original sketch had no coordination between a",
|
||||
"cache fill and a write and omitted the single-flight wrapper and the absence",
|
||||
"sentinel; finding F1 showed that violates the read-after-write rule. The",
|
||||
"ordering rules below replace it. `flight` is the existing per-key single-flight",
|
||||
"wrapper extended with `invalidate(key)` and `invalidateAll()`; a fill ticket is",
|
||||
"`live()` until its key is invalidated. `ProfileNotFound` stands for the",
|
||||
"repository's existing typed not-found error class.",
|
||||
].join('\n');
|
||||
test('accepts the actual finding without requiring its separate execution diagram', () => {
|
||||
expect(hasStaleFillRaceFinding(finding)).toBe(true);
|
||||
expect(hasStaleFillRaceFinding('## Proposed wrapper integration\nKeep the current read-through repository interface and shared adapters.\n' + finding)).toBe(true);
|
||||
});
|
||||
test.each([
|
||||
['current wrapper', finding.replace('original sketch had', 'current wrapper has')],
|
||||
['proposed implementation', finding.replace('original sketch had', 'proposed implementation has')],
|
||||
['freshness contract', finding.replace('read-after-write rule', 'read-after-write contract')],
|
||||
])('recognizes equivalent %s evidence', (_name, text) => expect(hasStaleFillRaceFinding(text)).toBe(true));
|
||||
test.each([
|
||||
['no violation asserted', finding.replace('finding F1 showed that violates the read-after-write rule.', '')],
|
||||
['negated violation', finding.replace('that violates', 'that does not violate')],
|
||||
['uncertain violation', finding.replace('that violates', 'that might violate')],
|
||||
['conditional premise', 'If ' + finding],
|
||||
['coordination exists', finding.replace('had no coordination', 'had coordination')],
|
||||
['different operations', finding.replace('cache fill and a write', 'cache hit and a read')],
|
||||
['wrong contract', finding.replace('read-after-write', 'read-before-write')],
|
||||
['dismissed defect', finding + '\n\nThis is not a defect; no fix is required.'],
|
||||
['accepted stale consequence', finding + '\n\nA subsequent stale read is permitted by the amended contract.'],
|
||||
['quoted finding', finding.split('\n').map(line => '> ' + line).join('\n')],
|
||||
['indented finding', finding.split('\n').map(line => ' ' + line).join('\n')],
|
||||
['quoted paragraph', '"' + finding + '"'],
|
||||
['source preface', 'Example of report format:\n' + finding],
|
||||
['separate source preface', 'Example of report format:\n\n' + finding],
|
||||
['backtick source fence', '````text\n' + finding + '\n````'],
|
||||
['tilde source fence', '~~~text\n' + finding + '\n~~~'],
|
||||
['unclosed source fence', '```text\n' + finding],
|
||||
['split unrelated paragraphs', finding.replace('sentinel; finding', 'sentinel.\n\n### Separate issue\nFinding')],
|
||||
])('rejects %s', (_name, text) => expect(hasStaleFillRaceFinding(text)).toBe(false));
|
||||
});
|
||||
|
||||
// Peer counterexamples: embedded, uncertain and hypothetical assertions stay closed.
|
||||
describe('coordination findings require directly asserted premises and conclusions', () => {
|
||||
const premise = 'The original sketch had no coordination between a cache fill and a write. ';
|
||||
const claim = 'This violates the read-after-write rule.';
|
||||
test('accepts a direct assertion', () => expect(hasStaleFillRaceFinding(premise + claim)).toBe(true));
|
||||
test.each([
|
||||
['negated embedded conclusion', premise + 'It is false that this violates the read-after-write rule.'],
|
||||
['unproven conclusion', premise + 'We have not shown that it violates the read-after-write rule.'],
|
||||
['uncertain conclusion', premise + 'It is unclear whether this violates the read-after-write rule.'],
|
||||
['question rather than assertion', premise + claim.replace('.', '?')],
|
||||
['separate issue without blank line', premise + '\n## A different issue\nReplica lag is high. ' + claim],
|
||||
['hypothetical premise', 'Suppose ' + premise + claim],
|
||||
['suggested report', 'A suggested report sentence: ' + premise + claim],
|
||||
['nested unmatched fence', '````text\n' + premise + claim + '\n```\n' + premise + claim + '\n````'],
|
||||
['mismatched fence', '~~~text\n' + premise + claim + '\n```\n' + premise + claim],
|
||||
])('rejects %s', (_name, text) => expect(hasStaleFillRaceFinding(text)).toBe(false));
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
import {expect,test} from 'bun:test';
|
||||
import captured from './fixtures/design-count-ad-v2.json';
|
||||
import {planCountQuestionPhase,designStep0Boundary,nativePlanCallFingerprint} from './helpers/claude-pty-runner';
|
||||
import {isDesignCountFirstReview,isDesignCountSetup} from './helpers/design-count-review';
|
||||
test('actual completed ordinary design Issue starts review at the finding, without counting later Eng work',()=>{
|
||||
expect(isDesignCountFirstReview(captured.firstFinding)).toBe(true);
|
||||
expect(planCountQuestionPhase(captured.firstFinding,false,designStep0Boundary,isDesignCountFirstReview,isDesignCountSetup)).toMatchObject({preReview:false,reviewStarted:true});
|
||||
});
|
||||
|
||||
test('ordinary design finding retains completed native identity and unresolved alternatives',()=>{
|
||||
for(const update of [
|
||||
(f:any)=>{f.nativeCall.answered=false;},(f:any)=>{f.nativeCall.failed=true;},
|
||||
(f:any)=>{f.signature='foreign';},(f:any)=>{f.nativeCall.unansweredQuestionIndices=[0];},
|
||||
(f:any)=>{f.nativeCall.answers={};},(f:any)=>{f.nativeCall.questions[0].header='Issue 2';},
|
||||
(f:any)=>{f.nativeCall.questions[0].multiSelect=true;},
|
||||
(f:any)=>{f.nativeCall.questions[0].question='Example: '+f.nativeCall.questions[0].question;},
|
||||
(f:any)=>{f.options.reverse();},
|
||||
]){const f=structuredClone(captured.firstFinding);update(f);expect(isDesignCountFirstReview(f)).toBe(false);}
|
||||
const f=structuredClone(captured.firstFinding);const q=f.nativeCall.questions[0]!;const old=q.question;
|
||||
q.question=q.question.replace('D4 — ','D38: ');f.nativeCall.answers={[q.question]:f.nativeCall.answers[old]!};
|
||||
expect(isDesignCountFirstReview(f)).toBe(true);
|
||||
});
|
||||
|
||||
test('optional gap tags do not decide the finding boundary',()=>{
|
||||
for(const replacement of [' (Visual Hierarchy)', '']){const f=structuredClone(captured.firstFinding),q=f.nativeCall.questions[0]!,old=q.question;q.question=q.question.replace(' (G1, Visual Hierarchy)',replacement);f.nativeCall.answers={[q.question]:f.nativeCall.answers[old]!};expect(isDesignCountFirstReview(f)).toBe(true);}
|
||||
});
|
||||
import {E2E_TOUCHFILES,selectTests} from './helpers/touchfiles';
|
||||
test('the retained Design calls select the native cadence workflow',()=>{
|
||||
for(const file of ['test/design-count-ad-v2.test.ts','test/fixtures/design-count-ad-v2.json'])
|
||||
expect(selectTests([file],E2E_TOUCHFILES,[]).selected).toEqual(['plan-design-finding-count']);
|
||||
});
|
||||
|
||||
test('an Issue label for participation or next-review routing is still setup',()=>{
|
||||
for(const [title,labels,description] of [
|
||||
['D4 — Issue 1: how should we address optional outside-review participation?', ['Run outside voices','Defer outside voices'],'Applies independent review to the plan.'],
|
||||
['D4 — Issue 1: how should we resolve which review runs next?', ['Run the engineering review','Defer next reviews'],'Closes the required engineering review gate.'],
|
||||
] as const){const c=structuredClone(captured.firstFinding.nativeCall),q=c.questions[0]!;q.question=title;q.options=labels.map((label,i)=>({label:`1${i?'B':'A'}) ${label}`,description}));c.answers={[title]:q.options[0]!.label};expect(isDesignCountFirstReview(nativePlanCallFingerprint(c,0,true))).toBe(false);}
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
import {expect, test} from 'bun:test';
|
||||
import fixture from './fixtures/devex-reconfirmation-ad-v2.json';
|
||||
import {isDevexReviewIssue} from './helpers/devex-count-fixture';
|
||||
import {nativePlanCallFingerprint} from './helpers/claude-pty-runner';
|
||||
import type {NativePlanQuestionCall} from './helpers/plan-count-transcript';
|
||||
const calls = () => structuredClone(fixture.calls) as NativePlanQuestionCall[];
|
||||
const fp = (c: NativePlanQuestionCall) => nativePlanCallFingerprint(c, 0, true);
|
||||
const classify = (c: NativePlanQuestionCall, history?: readonly NativePlanQuestionCall[]) => isDevexReviewIssue(fp(c), history);
|
||||
const answer = (c: NativePlanQuestionCall) => {c.answers = {[c.questions[0]!.question]:c.questions[0]!.options[0]!.label}; return c;};
|
||||
|
||||
test('completed roleplay reconfirmation adds no eighth issue after the five actual approvals', () => {
|
||||
const all = calls();
|
||||
expect(classify(all[9]!, all.slice(0, 9))).toBe(false);
|
||||
expect(all.filter((c,i) => classify(c, all.slice(0,i)))).toHaveLength(7);
|
||||
expect(fixture.provenance.actualOutcome).toBe('ceiling_reached');
|
||||
expect(fixture.provenance.noRetroactivePass).toBe(true);
|
||||
});
|
||||
|
||||
test('the five original issues plus later measurement and TODO decisions remain substantive', () => {
|
||||
const all = calls();
|
||||
for (const i of [4,5,6,7,8,11,12]) expect(classify(all[i]!, all.slice(0,i))).toBe(true);
|
||||
});
|
||||
|
||||
test('recap text alone cannot stand in for earlier completed same-session approvals', () => {
|
||||
const all = calls(), recap = all[9]!;
|
||||
expect(classify(recap)).toBe(true);
|
||||
expect(classify(recap, [])).toBe(true);
|
||||
for (const mutation of [
|
||||
(h: NativePlanQuestionCall[]) => h.splice(4,1),
|
||||
(h: NativePlanQuestionCall[]) => {h[4]!.sessionId='foreign';},
|
||||
(h: NativePlanQuestionCall[]) => {h[4]!.answered=false;},
|
||||
(h: NativePlanQuestionCall[]) => {h[4]!.failed=true;},
|
||||
(h: NativePlanQuestionCall[]) => {h[4]!.unansweredQuestionIndices=[0];},
|
||||
(h: NativePlanQuestionCall[]) => {h[4]!.answers={[h[4]!.questions[0]!.question]:h[4]!.questions[0]!.options.at(-1)!.label};},
|
||||
(h: NativePlanQuestionCall[]) => {h[4]!.answeredAt=recap.answeredAt;},
|
||||
(h: NativePlanQuestionCall[]) => {h[4]!.answeredAt='invalid';},
|
||||
]) {const h=all.slice(0,9).map(c=>structuredClone(c));mutation(h);expect(classify(recap,h)).toBe(true);}
|
||||
});
|
||||
|
||||
test('a new repair in the recap or selected choice remains a substantive decision', () => {
|
||||
for (const text of ['Add a new credential wizard.', 'Disable authentication.', 'Repair the retry assertion.', 'The plan must add a new endpoint.']) {
|
||||
for (const place of ['tail','inside-body','selected-description','selected-label'] as const) {
|
||||
const all=calls(), c=all[9]!, q=c.questions[0]!;
|
||||
if(place==='tail') q.question+='\n'+text;
|
||||
if(place==='inside-body') q.question=q.question.replace('\nELI10:', '\n'+text+'\nELI10:');
|
||||
if(place==='selected-description') q.options[0]!.description+=' '+text;
|
||||
if(place==='selected-label') q.options[0]!.label+=' '+text;
|
||||
expect(classify(answer(c),all.slice(0,9))).toBe(true);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('accuracy source, recap, and new measurement each keep distinct attribution', () => {
|
||||
const all=calls();
|
||||
expect(classify(all[3]!,all.slice(0,3))).toBe(false);
|
||||
expect(classify(all[9]!,all.slice(0,9))).toBe(false);
|
||||
expect(classify(all[11]!,all.slice(0,11))).toBe(true);
|
||||
expect(classify(all[12]!,all.slice(0,12))).toBe(true);
|
||||
expect(all.map((c,i)=>classify(c,all.slice(0,i)))).toEqual([
|
||||
false,false,false,false,true,true,true,true,true,false,false,true,true,
|
||||
]);
|
||||
});
|
||||
|
||||
test('accuracy explanations or choices cannot authorize an additional repair', () => {
|
||||
for(const text of ['Add a new credential wizard.','Disable authentication.','Repair the retry assertion.']) {
|
||||
for(const place of ['tail','outside-quote','description','label'] as const) {
|
||||
const all=calls(),c=all[3]!,q=c.questions[0]!;
|
||||
if(place==='tail')q.question+='\n'+text;
|
||||
if(place==='outside-quote')q.question=q.question.replace('\n\nELI10:', '\n\n'+text+'\n\nELI10:');
|
||||
if(place==='description')q.options[0]!.description+=' '+text;
|
||||
if(place==='label')q.options[0]!.label+=' '+text;
|
||||
expect(classify(answer(c),all.slice(0,3))).toBe(true);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('missing measurement must be an actual completed decision about a new release gate', () => {
|
||||
for(const mutate of [
|
||||
(c:NativePlanQuestionCall)=>{c.questions[0]!.question='Example: '+c.questions[0]!.question;answer(c);},
|
||||
(c:NativePlanQuestionCall)=>{c.questions[0]!.question=c.questions[0]!.question.replace('never re-measured','already re-measured');answer(c);},
|
||||
(c:NativePlanQuestionCall)=>{c.questions[0]!.question=c.questions[0]!.question.replace('Nothing in the plan re-runs','The existing plan already re-runs');answer(c);},
|
||||
(c:NativePlanQuestionCall)=>{c.answered=false;},
|
||||
(c:NativePlanQuestionCall)=>{c.failed=true;},
|
||||
(c:NativePlanQuestionCall)=>{c.unansweredQuestionIndices=[0];},
|
||||
(c:NativePlanQuestionCall)=>{c.answers={[c.questions[0]!.question]:'unoffered'};},
|
||||
(c:NativePlanQuestionCall)=>{c.questions[0]!.multiSelect=true;},
|
||||
]) {const c=calls()[11]!;mutate(c);expect(classify(c)).toBe(false);}
|
||||
});
|
||||
|
||||
test('history identities cannot be replaced by similarly labelled unapproved evidence', () => {
|
||||
for(const mutate of [
|
||||
(h:NativePlanQuestionCall[])=>{h[4]!.questions[0]!.question=h[4]!.questions[0]!.question.replace('D4','D40');answer(h[4]!);},
|
||||
(h:NativePlanQuestionCall[])=>{h[4]!.questions[0]!.multiSelect=true;},
|
||||
(h:NativePlanQuestionCall[])=>{h[4]!.answers={[h[4]!.questions[0]!.question]:'Fix in plan: unoffered new action'};},
|
||||
(h:NativePlanQuestionCall[])=>{h.push(structuredClone(h[4]!));},
|
||||
]) {const all=calls(),h=all.slice(0,9);mutate(h);expect(classify(all[9]!,h)).toBe(true);}
|
||||
const all=calls();
|
||||
expect(isDevexReviewIssue({...fp(all[9]!),signature:'foreign:call'},all.slice(0,9))).toBe(true);
|
||||
});
|
||||
|
||||
test('the same subject with a different approved change cannot establish the recapped repair', () => {
|
||||
const cases = [
|
||||
(c: NativePlanQuestionCall) => {
|
||||
c.questions[0]!.question = 'D4 — Add debug logging to examples/first_eval.py?';
|
||||
c.questions[0]!.options[0] = {label:'Fix in plan: add debug logging',description:'Add diagnostics without changing which files ship.'};
|
||||
},
|
||||
(c: NativePlanQuestionCall) => {
|
||||
c.questions[0]!.options[0] = {label:'Fix in plan: document the missing example without shipping it',description:'Document the absent file; do not ship or replace it.'};
|
||||
},
|
||||
];
|
||||
for (const change of cases) {
|
||||
const all = calls(); change(all[4]!); answer(all[4]!);
|
||||
expect(classify(all[9]!, all.slice(0,9))).toBe(true);
|
||||
}
|
||||
for (let i=4; i<=8; i++) {
|
||||
const all = calls();
|
||||
all[i]!.questions[0]!.options[0]!.description = 'Document the existing behavior; leave the runtime and published contracts unchanged.';
|
||||
answer(all[i]!);
|
||||
expect(classify(all[9]!,all.slice(0,9))).toBe(true);
|
||||
const more = calls();
|
||||
more[i]!.questions[0]!.options[0]!.description += ' Also disable authentication.';
|
||||
answer(more[i]!);
|
||||
expect(classify(more[9]!,more.slice(0,9))).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
import {E2E_TOUCHFILES,selectTests} from './helpers/touchfiles';
|
||||
test('DX native evidence and history integration select its paid workflow',()=>{
|
||||
for(const file of ['test/devex-reconfirmation-ad-v2.test.ts','test/fixtures/devex-reconfirmation-ad-v2.json','test/plan-count-history.test.ts'])
|
||||
expect(selectTests([file],E2E_TOUCHFILES).selected).toContain('plan-devex-finding-count');
|
||||
});
|
||||
@@ -253,3 +253,76 @@ describe('disabled outside-plan live oracle', () => {
|
||||
}, 150_000);
|
||||
|
||||
});
|
||||
|
||||
|
||||
// Both actual attempts obeyed the off switch; prior records and rejected claims
|
||||
// were mistaken for current completion by the bare substring check.
|
||||
describe('AD v2 disabled-plan public attribution', () => {
|
||||
const captured = require('./fixtures/disabled-plan-attribution-ad-v2.json');
|
||||
test.each(captured.cases)('accepts actual attempt $attempt output without crediting historical coverage', (item: any) => {
|
||||
expect(item.actualOracle).toMatchObject({ passed: false, falseCompletion: true, completed: true,
|
||||
agentAvailable: true, disabledAttribution: true, persistedDisabled: true, fallbackCalls: [], cliDispatchLog: '' });
|
||||
const result = completed(); result.output = item.output; result.transcript.at(-1).result = item.output;
|
||||
expect(disabledPlanReviewEvidence(result, '', item.reviewRecords.map((record: any) => JSON.stringify(record)).join('\n'), item.reviewRecords[0]))
|
||||
.toMatchObject({ passed: true, falseCompletion: false, completed: true, persistedDisabled: true });
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe('disabled attribution keeps denial and history local to each mention', () => {
|
||||
const withOutput = (text: string) => { const result = completed(); result.output += '\n' + text; return result; };
|
||||
test.each([
|
||||
'There is no "both reviewers agree" claim either.',
|
||||
'We do not claim both reviewers agree.',
|
||||
"I cannot claim 'both reviewers agree'.",
|
||||
'I am not claiming “both reviewers agree”.',
|
||||
'No “\'both reviewers agree\'” claim was made.',
|
||||
'The review log held an earlier entry, timestamped two minutes before this run, claiming `outside_status: completed`.',
|
||||
'A prior record shows outside_status: completed. It predates this workflow and supplies no current coverage.',
|
||||
'Historical record: outside_status: completed. Current outside_status: disabled.',
|
||||
])('does not convert an explicit denial or historical record into coverage: %s', text => {
|
||||
expect(oracle(withOutput(text))).toMatchObject({ passed: true, falseCompletion: false });
|
||||
});
|
||||
test.each([
|
||||
'Both reviewers agree.',
|
||||
'outside_status: completed',
|
||||
'"Both reviewers agree."',
|
||||
'Possibly both reviewers agree.',
|
||||
'We cannot confirm outside_status: completed.',
|
||||
'Outside_status: completed?',
|
||||
'No "both reviewers agree" claim earlier. Both reviewers agree now.',
|
||||
'We do not claim both reviewers agree; outside_status: completed.',
|
||||
'No “\'both reviewers agree\'” claim; but both reviewers agree.',
|
||||
'The prior record shows outside_status: completed. Current outside_status: completed.',
|
||||
'The earlier entry claims outside_status: completed; outside_status: completed.',
|
||||
'The earlier entry claims outside_status: completed, but outside_status: completed.',
|
||||
'The prior record was updated now to outside_status: completed.',
|
||||
'A prior record shows outside_status: completed, and we report outside_status: completed.',
|
||||
'A prior record was inspected, and we report outside_status: completed.',
|
||||
'A prior record was inspected, and the model reports outside_status: completed.',
|
||||
'A prior record shows successful validation, and the model reports outside_status: completed.',
|
||||
'The prior record shows success and the reviewer concludes outside_status: completed.',
|
||||
'A prior record shows outside_status: completed, and reports outside_status: completed.',
|
||||
'The earlier record shows outside_status: completed and both reviewers agree.',
|
||||
'The earlier record shows outside_status: completed and this run has outside_status: completed.',
|
||||
'The historical record shows outside_status: completed.\n```json\n{"outside_status":"completed"}\n```',
|
||||
])('rejects current or uncertain completion language even beside a safe mention: %s', text => {
|
||||
expect(oracle(withOutput(text))).toMatchObject({ passed: false, falseCompletion: true });
|
||||
});
|
||||
test('safe public attribution cannot override actual dispatch or missing execution/persistence', () => {
|
||||
const captured = require('./fixtures/disabled-plan-attribution-ad-v2.json');
|
||||
for (const item of captured.cases) {
|
||||
const result = withOutput(item.output);
|
||||
expect(oracle(result, 'codex invoked\n').passed).toBe(false);
|
||||
for (const tool of ['Agent', 'Task']) {
|
||||
const dispatched = structuredClone(result); dispatched.transcript.splice(-1, 0, dispatch(tool, { prompt: 'Outside review' }));
|
||||
expect(oracle(dispatched)).toMatchObject({ passed: false, falseCompletion: false });
|
||||
}
|
||||
const noPreflight = structuredClone(result); noPreflight.transcript.splice(1, 2);
|
||||
expect(oracle(noPreflight).passed).toBe(false);
|
||||
expect(disabledPlanReviewEvidence(result, '', JSON.stringify(PRIOR_RECORD), PRIOR_RECORD).passed).toBe(false);
|
||||
const noCompletion = structuredClone(result); noCompletion.transcript.pop();
|
||||
expect(oracle(noCompletion).passed).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,188 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import captured from './fixtures/eng-count-ad-v2.json';
|
||||
import { engFirstReviewAUQ, engSetupAUQ, engStep0Boundary, nativePlanCallFingerprint, planCountQuestionPhase } from './helpers/claude-pty-runner';
|
||||
import type { NativePlanQuestionCall } from './helpers/plan-count-transcript';
|
||||
import { isEngCompletionHandoff } from './helpers/eng-completion-handoff';
|
||||
import { E2E_TOUCHFILES, matchGlob } from './helpers/touchfiles';
|
||||
|
||||
const firstCalls = captured.cases.first.calls as NativePlanQuestionCall[];
|
||||
const retryCalls = captured.cases.retry.calls as NativePlanQuestionCall[];
|
||||
const catalog = captured.reviewedTasks.lines.join('\n');
|
||||
const issue = () => structuredClone(retryCalls[3]!);
|
||||
const handoff = () => structuredClone(firstCalls.at(-1)!);
|
||||
const fp = (call: NativePlanQuestionCall) => nativePlanCallFingerprint(call, 0, true);
|
||||
const isFirst = (call: NativePlanQuestionCall) => engFirstReviewAUQ(fp(call));
|
||||
const isHandoff = (call: NativePlanQuestionCall, plan = catalog) => isEngCompletionHandoff(fp(call), plan);
|
||||
function changeQuestion(call: NativePlanQuestionCall, change: (s: string) => string) {
|
||||
const q = call.questions[0]!, answer = call.answers?.[q.question];
|
||||
q.question = change(q.question); call.answers = answer ? { [q.question]: answer } : {}; return call;
|
||||
}
|
||||
function census(calls: NativePlanQuestionCall[]) {
|
||||
let reviewStarted = false;
|
||||
const counts = { setup: 0, review: 0, administrative: 0 };
|
||||
const phases = calls.map(call => {
|
||||
const phase = planCountQuestionPhase(fp(call), reviewStarted, engStep0Boundary, engFirstReviewAUQ, engSetupAUQ,
|
||||
current => isEngCompletionHandoff(current, catalog));
|
||||
reviewStarted = phase.reviewStarted;
|
||||
counts[phase.administrative ? 'administrative' : phase.preReview ? 'setup' : 'review']++;
|
||||
return phase;
|
||||
});
|
||||
return { counts, phases };
|
||||
}
|
||||
|
||||
describe('Eng AD v2 completed native count evidence', () => {
|
||||
test('first attempt retains seven substantive decisions and separates the completed D9 handoff', () => {
|
||||
const { counts, phases } = census(firstCalls);
|
||||
expect(counts).toEqual({ setup: 4, review: 7, administrative: 1 });
|
||||
expect(phases.slice(4, 11).every(p => !p.preReview && !p.administrative)).toBe(true);
|
||||
expect(firstCalls[9]!.questions[0]!.question).toContain('TODO 1');
|
||||
expect(firstCalls[10]!.questions[0]!.question).toContain('TODO 2');
|
||||
expect(phases[11]!.administrative).toBe('completion-handoff');
|
||||
expect(captured.cases.first.actual.outcome).toBe('ceiling_reached');
|
||||
expect(captured.cases.first.actual.reviewCount).toBe(8);
|
||||
});
|
||||
|
||||
test('retry ordinary Issue identity starts review without qids, retaining its later TODO', () => {
|
||||
const { counts, phases } = census(retryCalls);
|
||||
expect(counts).toEqual({ setup: 3, review: 6, administrative: 0 });
|
||||
expect(phases.slice(3).every(p => !p.preReview && !p.administrative)).toBe(true);
|
||||
for (const call of retryCalls.slice(3, 8)) expect(isFirst(call)).toBe(true);
|
||||
expect(retryCalls[8]!.questions[0]!.question).toContain('TODO 1');
|
||||
expect(captured.cases.retry.actual.reviewCount).toBe(0);
|
||||
});
|
||||
|
||||
test('the prior successful plan Write already contains the exact referenced task and regression step', () => {
|
||||
expect(captured.reviewedTasks.isError).toBe(false);
|
||||
expect(Date.parse(captured.reviewedTasks.replyAt)).toBeLessThan(Date.parse(handoff().answeredAt!));
|
||||
expect(captured.reviewedTasks.lines).toHaveLength(10);
|
||||
expect(captured.reviewedTasks.lines[2]).toContain('Record regression characterization fixtures before any change');
|
||||
expect(isHandoff(handoff())).toBe(true);
|
||||
// The menu's Tasks JSONL claim is not independently verified by this fixture.
|
||||
expect(captured.provenance.privateThinkingInspected).toBe(false);
|
||||
});
|
||||
|
||||
test('ordinary issue presentation can vary while completed identity and section number remain bound', () => {
|
||||
for (const title of ['Issue 1', 'Finding 1.2 (D17)', 'D42 — Issue 1']) {
|
||||
const call = changeQuestion(issue(), s => s.replace('Issue 1 (D4)', title).replace('AuthCache', 'SessionCache'));
|
||||
call.questions[0]!.header = title.includes('1.2') ? 'Architecture 1.2' : 'Architecture 1';
|
||||
call.questions[0]!.options.reverse();
|
||||
for (const option of call.questions[0]!.options) {
|
||||
call.answers = { [call.questions[0]!.question]: option.label };
|
||||
expect(isFirst(call)).toBe(true);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('setup, quoted or mismatched section identities do not start review', () => {
|
||||
for (const header of ['Scope', 'Approach', 'Next steps', 'Arch 2', 'Example Arch 1', 'TODO 1']) {
|
||||
const call = issue(); call.questions[0]!.header = header; expect(isFirst(call)).toBe(false);
|
||||
}
|
||||
for (const change of [
|
||||
(s: string) => '> ' + s,
|
||||
(s: string) => 'Example: ' + s,
|
||||
(s: string) => '```text\n' + s + '\n```',
|
||||
(s: string) => s.replace('Issue 1 (D4)', 'Issue 2 (D4)'),
|
||||
(s: string) => s + ' <gstack-qid:plan-eng-setup>',
|
||||
]) expect(isFirst(changeQuestion(issue(), change))).toBe(false);
|
||||
for (const call of [...firstCalls.slice(0, 4), ...retryCalls.slice(0, 3)]) expect(isFirst(call)).toBe(false);
|
||||
});
|
||||
|
||||
test('an Issue heading alone cannot turn a confirmation or report action into a finding', () => {
|
||||
for (const body of [
|
||||
'No defect remains in the cache. Proceed with the next section?',
|
||||
'The cache already serializes writes. Confirm this is accurate?',
|
||||
'Should I save the reviewed plan now?',
|
||||
'Add a section to the reviewed plan?',
|
||||
'Serialize the reviewed plan as JSON for the handoff?',
|
||||
'Add the completed tests to this report?',
|
||||
]) {
|
||||
const c = changeQuestion(issue(), () => 'Issue 1 (D4) — ' + body);
|
||||
c.questions[0]!.options = [
|
||||
{ label: 'Yes', description: 'Confirm this statement; no new implementation work.' },
|
||||
{ label: 'No', description: 'Do not confirm; no new implementation work.' },
|
||||
];
|
||||
c.answers = { [c.questions[0]!.question]: 'Yes' };
|
||||
expect(isFirst(c)).toBe(false);
|
||||
}
|
||||
const c = issue();
|
||||
c.questions[0]!.options = [{ label: 'Yes', description: 'Confirm; no new work.' }, { label: 'No', description: 'Decline; no new work.' }];
|
||||
c.answers = { [c.questions[0]!.question]: 'Yes' };
|
||||
expect(isFirst(c)).toBe(false);
|
||||
});
|
||||
|
||||
test('new first-finding and handoff paths require exact completed native identity and answer', () => {
|
||||
for (const factory of [issue, handoff]) {
|
||||
const classify = factory === issue ? isFirst : isHandoff;
|
||||
for (const mutate of [
|
||||
(c: NativePlanQuestionCall) => { c.answered = false; },
|
||||
(c: NativePlanQuestionCall) => { c.failed = true; },
|
||||
(c: NativePlanQuestionCall) => { delete c.failed; },
|
||||
(c: NativePlanQuestionCall) => { c.sessionId = ''; },
|
||||
(c: NativePlanQuestionCall) => { c.toolUseId = ''; },
|
||||
(c: NativePlanQuestionCall) => { delete c.answeredAt; },
|
||||
(c: NativePlanQuestionCall) => { c.answeredAt = 'invalid'; },
|
||||
(c: NativePlanQuestionCall) => { c.unansweredQuestionIndices = [0]; },
|
||||
(c: NativePlanQuestionCall) => { delete c.unansweredQuestionIndices; },
|
||||
(c: NativePlanQuestionCall) => { c.questions[0]!.multiSelect = true; },
|
||||
(c: NativePlanQuestionCall) => { c.questions.push(structuredClone(c.questions[0]!)); },
|
||||
(c: NativePlanQuestionCall) => { c.questions[0]!.options[1]!.label = c.questions[0]!.options[0]!.label; },
|
||||
(c: NativePlanQuestionCall) => { c.answers = {}; },
|
||||
(c: NativePlanQuestionCall) => { c.answers = { [c.questions[0]!.question]: 'Unoffered' }; },
|
||||
(c: NativePlanQuestionCall) => { c.answers!.foreign = 'Foreign'; },
|
||||
]) { const c = factory(); mutate(c); expect(classify(c)).toBe(false); }
|
||||
const classifyFp = factory === issue ? engFirstReviewAUQ : (f: ReturnType<typeof fp>) => isEngCompletionHandoff(f, catalog);
|
||||
expect(classifyFp({ ...fp(factory()), signature: 'foreign:call' })).toBe(false);
|
||||
expect(classifyFp({ ...fp(factory()), nativeCall: undefined })).toBe(false);
|
||||
expect(classifyFp({ ...fp(factory()), nativeQuestionIndex: 1 })).toBe(false);
|
||||
expect(classifyFp({ ...fp(factory()), options: [] })).toBe(false);
|
||||
const wrong = fp(factory()); wrong.options[0]!.index = 2; expect(classifyFp(wrong)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('closed handoff accepts either offered action and order, but cannot start or satisfy a review', () => {
|
||||
const call = handoff(); call.questions[0]!.options.reverse();
|
||||
for (const option of call.questions[0]!.options) {
|
||||
call.answers = { [call.questions[0]!.question]: option.label };
|
||||
expect(isHandoff(call)).toBe(true);
|
||||
}
|
||||
expect(census([call]).counts).toEqual({ setup: 0, review: 0, administrative: 1 });
|
||||
expect(census([call]).phases[0]!.reviewStarted).toBe(false);
|
||||
});
|
||||
|
||||
test('new task references or a missing, contradicted, or incomplete reviewed catalog remain substantive', () => {
|
||||
for (const plan of ['', catalog.replace('**T10 ', '**T11 '), catalog + '\n' + captured.reviewedTasks.lines[2],
|
||||
catalog.replace('Record regression', 'Do not record regression'), catalog.replace('Record regression', 'Discuss regression')]) {
|
||||
expect(isHandoff(handoff(), plan)).toBe(false);
|
||||
}
|
||||
for (const change of [
|
||||
(s: string) => s.replace('T1–T10', 'T1–T11'),
|
||||
(s: string) => s.replace('T1–T10', 'T2–T10'),
|
||||
(s: string) => s.replace('record T3', 'record T4'),
|
||||
(s: string) => s + ' Also add a new migration before shipping.',
|
||||
(s: string) => s.replace('implement T1–T10', 'approve and implement T1–T10'),
|
||||
]) { const c = handoff(); c.questions[0]!.options[0]!.description = change(c.questions[0]!.options[0]!.description); expect(isHandoff(c)).toBe(false); }
|
||||
});
|
||||
|
||||
test('conditional closure, extra decisions, appended new work and quoted navigation are never discounted', () => {
|
||||
for (const change of [
|
||||
(s: string) => s.replace('Eng Review is CLEAR', 'Eng Review will be CLEAR after fixing the race'),
|
||||
(s: string) => s.replace('Eng Review is CLEAR', 'Eng Review is not CLEAR'),
|
||||
(s: string) => s.replace('What next?', 'What next? Also approve deleting the migration?'),
|
||||
(s: string) => s + '\nCreate another cache before the next review.',
|
||||
(s: string) => '> ' + s,
|
||||
(s: string) => '```text\n' + s + '\n```',
|
||||
]) expect(isHandoff(changeQuestion(handoff(), change))).toBe(false);
|
||||
const c = handoff(); c.questions[0]!.options[1]!.description += ' Remove the CI gate first.'; expect(isHandoff(c)).toBe(false);
|
||||
const label = handoff(); label.questions[0]!.options[0]!.label += ' and rewrite auth';
|
||||
label.answers = { [label.questions[0]!.question]: label.questions[0]!.options[0]!.label }; expect(isHandoff(label)).toBe(false);
|
||||
const header = handoff(); header.questions[0]!.header = 'Issue 9'; expect(isHandoff(header)).toBe(false);
|
||||
});
|
||||
|
||||
test('new evidence selects precisely its affected existing paid workflows', () => {
|
||||
const selected = (path: string) => Object.entries(E2E_TOUCHFILES).filter(([, patterns]) => patterns.some(p => matchGlob(path, p))).map(([name]) => name).sort();
|
||||
for (const path of ['test/eng-count-ad-v2.test.ts', 'test/fixtures/eng-count-ad-v2.json']) {
|
||||
expect(selected(path)).toEqual(['plan-eng-finding-count', 'plan-eng-multi-finding-batching']);
|
||||
}
|
||||
expect(selected('test/helpers/eng-completion-handoff.ts')).toEqual(['plan-eng-finding-count']);
|
||||
});
|
||||
});
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"provenance": {
|
||||
"kind": "Actual complete public setup packet and current visible first pane",
|
||||
"label": "ship-source-ad-full-paid-20260909-v2",
|
||||
"capturedAt": "2026-09-09T19:05:10.548Z",
|
||||
"sourceHead": "7fac51e9bd8619665960657462ceab2773d142f5",
|
||||
"observationSha256": "ef98690ff24589a177027e57b5159144d2e9269ff3b7ee706bd9f57f51f4b492",
|
||||
"screenSha256": "8ca0a32e2e894ba5dd4fad2c10e225f847e8a64792e7f0aeea13d6183e5b4503",
|
||||
"outcome": "live case waiting; no input/pass or retrospective coverage from fixture"
|
||||
},
|
||||
"pendingCall": {
|
||||
"sessionId": "e40afe62-b71e-4372-9ff1-981ced50de14",
|
||||
"toolUseId": "toolu_01RyKcb3VJtBWLqb3h56DPBJ",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D1 \u2014 Add gstack skill routing rules to CLAUDE.md?\nProject/branch/task: gstack-autoplan-chain on main, about to /autoplan the User Dashboard plan.\nELI10: gstack works best when your project's CLAUDE.md includes skill routing rules. This repo has no CLAUDE.md yet. Routing rules tell the agent which skill to reach for (bugs \u2192 /investigate, ship \u2192 /ship) so you don't have to name skills by hand. Stakes: without it, skills only run when you invoke them explicitly.\nStakes if we pick wrong: mild \u2014 either a small extra file in the repo, or slightly more manual skill invocation.\nRecommendation: A because routing rules are a one-time, low-cost file that makes every future session smoother.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: a committed CLAUDE.md section vs. manual skill invocation. Note: we are in plan mode, so the file write and commit happen after plan approval, not now.",
|
||||
"header": "Routing",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Add routing rules to CLAUDE.md (recommended)",
|
||||
"description": "\u2705 Future requests auto-route to the right skill without you naming it. \u2705 One-time setup, committed once, shared with teammates. \u274c Adds a CLAUDE.md file to a repo that has none today (written after plan approval)."
|
||||
},
|
||||
{
|
||||
"label": "No thanks, I'll invoke skills manually",
|
||||
"description": "\u2705 Repo stays minimal, no new files added. \u2705 You keep full control over when skills run. \u274c Skills never fire proactively; you type each one yourself."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"question": "D2 \u2014 Run /office-hours first for a design doc?\nProject/branch/task: same repo, no design doc found for this branch.\nELI10: No design doc found for this branch. /office-hours produces a structured problem statement, premise challenge, and explored alternatives \u2014 it gives this review much sharper input to work with. Takes about 10 minutes. The design doc is per-feature, not per-product \u2014 it captures the thinking behind this specific change. Stakes: the CEO review phase leans on a design doc to judge premises; without one it works from the plan text alone.\nStakes if we pick wrong: either 10 extra minutes up front, or a CEO review with slightly less context on why this dashboard exists.\nRecommendation: B because the plan already carries a detailed 'Existing product and application contracts' section with the problem (75s median to find next task), success metric (45s), and guardrails, which is most of what office-hours would produce.\nCompleteness: A=10/10, B=8/10\nNet: 10 minutes of upfront problem framing vs. proceeding on an already well-grounded plan.",
|
||||
"header": "Design doc",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Run /office-hours now",
|
||||
"description": "\u2705 Produces a formal problem statement and alternatives the CEO review can challenge directly. \u2705 Leaves a committed design doc for teammates. \u274c Adds ~10 minutes (human: ~30 min / CC: ~10 min) before the review starts, and this plan already states its problem and metric."
|
||||
},
|
||||
{
|
||||
"label": "Skip \u2014 proceed with standard review (recommended)",
|
||||
"description": "\u2705 Review starts immediately using the plan's existing problem statement and success metrics. \u2705 The plan's contracts section already covers most of what office-hours would surface. \u274c No separate design doc artifact; premise challenge works from plan text only."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": false,
|
||||
"failed": false
|
||||
},
|
||||
"screen": "\n\u25cf Bash(eval \"$(~/.claude/skills/gstack/bin/gstack-slug 2>/dev/null)\"\n echo \"SLUG=$SLUG BRANCH=$BRANCH\"\u2026)\n \u23bf \u00a0SLUG=gstack-autoplan-chain-f8hO0F BRANCH=main\n --- RECENT ARTIFACTS ---\n {\n \u2026 +32 lines (ctrl+o to expand)\n \u23bf \u00a0Allowed by auto mode classifier\n\n\u25cf Everything checks out \u2014 I found one plan file for a user dashboard page, but no design doc or CLAUDE.md. Two gates\n triggered before running: the routing-rules onboarding and the office-hours offer. \u00b7\u00a0summarized\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\nPlanning:\n/tmp/gstack-paid-shard-GJoH2D/tmp/gstack-hermetic-1636446-Otiddj/with-skills/.claude/plans/cheerful-moseying-robin.md\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\u2190 \u2610 Routing \u2610 Design doc \u2714 Submit \u2192\n\n\u2502 D1 \u2014 Add gstack skill routing rules to CLAUDE.md?\n\u2502 Project/branch/task: gstack-autoplan-chain on main, about to /autoplan the User Dashboard plan.\n\u2502 ELI10: gstack works best when your project's CLAUDE.md includes skill routing rules. This repo has no CLAUDE.md yet.\n\u2502 Routing rules tell the agent which skill to reach for (bugs \u2192 /investigate, ship \u2192 /ship) so you don't have to name\n\u2502 skills by hand. Stakes: without it, skills only run when you invoke them explicitly.\n\u2502 Stakes if we pick wrong: mild \u2014 either a small extra file in the repo, or slightly more manual skill invocation.\n\u2502 Recommendation: A because routing rules are a one-time, low-cost file that makes every future session smoother.\n\u2502 Note: options differ in kind, not coverage \u2014 no completeness score.\n\u2502 Net: a committed CLAUDE.md section vs. manual skill invocation. Note: we are in plan mode, so the file write and\n\u2502 commit happen after plan approval, not now.\n\n\u276f 1. Add routing rules to CLAUDE.md (recommended)\n \u2705 Future requests auto-route to the right skill without you naming it. \u2705 One-time setup, committed once, shared\n with teammates. \u274c Adds a CLAUDE.md file to a repo that has none today (written after plan approval).\n 2. No thanks, I'll invoke skills manually\n \u2705 Repo stays minimal, no new files added. \u2705 You keep full control over when skills run. \u274c Skills never fire\n proactively; you type each one yourself.\n 3. Type something.\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n 4. Chat about this\n\nEnter to select \u00b7 Tab/Arrow keys to navigate \u00b7 Esc to cancel\n"
|
||||
}
|
||||
Vendored
+2430
File diff suppressed because it is too large.
Load diff
+73
@@ -0,0 +1,73 @@
|
||||
{
|
||||
"provenance": {
|
||||
"sourceHead": "7fac51e9bd8619665960657462ceab2773d142f5",
|
||||
"sourceReceipt": ".context/ship-source-ad-full-paid-20260909-v2/mode-office-hours-header-evidence-ledger-v1/proof.json",
|
||||
"completedCallSHA256": "1cb969cf3ffe8debed713d7f698776ac3ea1f94f01e0b0521568e8d21512b67e",
|
||||
"requestedAt": "2026-09-09T19:06:48.582Z",
|
||||
"completedAt": "2026-09-09T19:07:02.700Z",
|
||||
"actualFirstAttempt": "mode_not_reached after420000ms",
|
||||
"laterRetry": "passed",
|
||||
"panes": "Tests project pending/active panes from exact complete public questions; no historical pending screen claim",
|
||||
"privateThinkingInspected": false
|
||||
},
|
||||
"completedCall": {
|
||||
"sessionId": "39bfb7b6-c281-4d16-ac5b-6a455c6e764a",
|
||||
"toolUseId": "toolu_01PRrHtcPwjYRtaEVL7HBqrD",
|
||||
"questions": [
|
||||
{
|
||||
"header": "Routing",
|
||||
"question": "D1 \u2014 Add gstack skill routing rules to CLAUDE.md?\nProject/branch/task: gstack-plan-count-FDFfod on main, reviewing PLAN.md (saved project views).\nELI10: gstack skills work best when the project's CLAUDE.md tells the agent which skill to reach for (bugs \u2192 /investigate, scope \u2192 /plan-ceo-review, etc.). Without it you invoke skills by hand each time. Stakes are low either way. Note: plan mode is active, so if you say yes the CLAUDE.md edit and commit happen after this review exits plan mode.\nStakes if we pick wrong: mild friction only; reversible with one config command.\nRecommendation: A because routing rules make future sessions pick the right skill without you remembering names.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) Add routing rules (recommended)\n \u2705 Future sessions auto-route bugs, scope, ship, and QA requests to the right skill\n \u2705 One-time, committed to the repo so teammates get it too\n \u274c Adds ~20 lines to CLAUDE.md; must wait until plan mode exits to write it\nB) No thanks\n \u2705 Keeps CLAUDE.md exactly as the fixture defines it right now\n \u2705 Nothing written, nothing committed; re-enable any time via gstack-config\n \u274c You invoke every skill manually going forward\nNet: convenience later vs an untouched CLAUDE.md now.",
|
||||
"options": [
|
||||
{
|
||||
"label": "Add routing rules (recommended)",
|
||||
"description": "Append the skill routing section to CLAUDE.md and commit once plan mode exits."
|
||||
},
|
||||
{
|
||||
"label": "No thanks",
|
||||
"description": "Set routing_declined=true; invoke skills manually."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
},
|
||||
{
|
||||
"header": "Learnings",
|
||||
"question": "D2 \u2014 Enable cross-project learnings search?\nProject/branch/task: same repo, same review.\nELI10: gstack records small lessons per project (command quirks, pitfalls). It can also search lessons from your other projects on this machine to spot patterns that apply here. Everything stays local; no data leaves the machine. Good for solo devs; skip if you juggle multiple client codebases where cross-talk would be awkward.\nStakes if we pick wrong: reversible with one config command; worst case a slightly less informed review.\nRecommendation: A because this is a solo fixture repo and more prior context only helps.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) Enable cross-project learnings (recommended)\n \u2705 Reviews get prior lessons from any project on this machine, not just this one\n \u2705 Stays fully local; a single config flag you can flip back any time\n \u274c Lessons from unrelated codebases may occasionally surface as noise\nB) Keep learnings project-scoped\n \u2705 Zero chance of one client's patterns leaking into another's review\n \u2705 Simpler mental model: what you see came from this repo only\n \u274c This repo has zero learnings today, so the review starts cold\nNet: broader context vs strict per-project isolation.",
|
||||
"options": [
|
||||
{
|
||||
"label": "Enable cross-project (recommended)",
|
||||
"description": "Search learnings across all local projects."
|
||||
},
|
||||
{
|
||||
"label": "Project-scoped only",
|
||||
"description": "Only this repo's learnings are used."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
},
|
||||
{
|
||||
"header": "Office hours",
|
||||
"question": "D3 \u2014 Run /office-hours first, or proceed with the standard review?\nProject/branch/task: same repo; no design doc exists for this branch.\nELI10: /office-hours produces a structured problem statement, a premise challenge, and explored alternatives for this specific feature. It gives the CEO review sharper input, at the cost of roughly ten more minutes of Q&A. The plan here is short (4 approach bullets), so the review can run without it, but the premise challenge will be based on what I infer rather than what you told me.\nStakes if we pick wrong: a review built on a misread premise, or ten minutes spent restating what you already know.\nRecommendation: B because the plan's goal is concrete and well-bounded; the review's Step 0 already runs a premise challenge, and I can offer office-hours mid-review if the premise turns out to be shaky.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) Run /office-hours now\n \u2705 Produces a written design doc that later reviews and teammates can read\n \u2705 Explores alternatives to saved views before we commit to reviewing this shape\n \u274c About ten minutes of additional questions before the review starts (human ~10 min / CC ~10 min)\nB) Skip, standard review (recommended)\n \u2705 Starts the review immediately on the plan as written\n \u2705 Step 0 still challenges the premise and proposes implementation alternatives\n \u274c No persisted design doc; premise reasoning rests on my inference from the plan text\nNet: sharper input later vs starting now on a plan that is already fairly clear.",
|
||||
"options": [
|
||||
{
|
||||
"label": "Run /office-hours now",
|
||||
"description": "Build a design doc first, then resume the review."
|
||||
},
|
||||
{
|
||||
"label": "Skip, standard review (recommended)",
|
||||
"description": "Proceed directly to Step 0 of the CEO review."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D1 \u2014 Add gstack skill routing rules to CLAUDE.md?\nProject/branch/task: gstack-plan-count-FDFfod on main, reviewing PLAN.md (saved project views).\nELI10: gstack skills work best when the project's CLAUDE.md tells the agent which skill to reach for (bugs \u2192 /investigate, scope \u2192 /plan-ceo-review, etc.). Without it you invoke skills by hand each time. Stakes are low either way. Note: plan mode is active, so if you say yes the CLAUDE.md edit and commit happen after this review exits plan mode.\nStakes if we pick wrong: mild friction only; reversible with one config command.\nRecommendation: A because routing rules make future sessions pick the right skill without you remembering names.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) Add routing rules (recommended)\n \u2705 Future sessions auto-route bugs, scope, ship, and QA requests to the right skill\n \u2705 One-time, committed to the repo so teammates get it too\n \u274c Adds ~20 lines to CLAUDE.md; must wait until plan mode exits to write it\nB) No thanks\n \u2705 Keeps CLAUDE.md exactly as the fixture defines it right now\n \u2705 Nothing written, nothing committed; re-enable any time via gstack-config\n \u274c You invoke every skill manually going forward\nNet: convenience later vs an untouched CLAUDE.md now.": "Add routing rules (recommended)",
|
||||
"D2 \u2014 Enable cross-project learnings search?\nProject/branch/task: same repo, same review.\nELI10: gstack records small lessons per project (command quirks, pitfalls). It can also search lessons from your other projects on this machine to spot patterns that apply here. Everything stays local; no data leaves the machine. Good for solo devs; skip if you juggle multiple client codebases where cross-talk would be awkward.\nStakes if we pick wrong: reversible with one config command; worst case a slightly less informed review.\nRecommendation: A because this is a solo fixture repo and more prior context only helps.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) Enable cross-project learnings (recommended)\n \u2705 Reviews get prior lessons from any project on this machine, not just this one\n \u2705 Stays fully local; a single config flag you can flip back any time\n \u274c Lessons from unrelated codebases may occasionally surface as noise\nB) Keep learnings project-scoped\n \u2705 Zero chance of one client's patterns leaking into another's review\n \u2705 Simpler mental model: what you see came from this repo only\n \u274c This repo has zero learnings today, so the review starts cold\nNet: broader context vs strict per-project isolation.": "Enable cross-project (recommended)",
|
||||
"D3 \u2014 Run /office-hours first, or proceed with the standard review?\nProject/branch/task: same repo; no design doc exists for this branch.\nELI10: /office-hours produces a structured problem statement, a premise challenge, and explored alternatives for this specific feature. It gives the CEO review sharper input, at the cost of roughly ten more minutes of Q&A. The plan here is short (4 approach bullets), so the review can run without it, but the premise challenge will be based on what I infer rather than what you told me.\nStakes if we pick wrong: a review built on a misread premise, or ten minutes spent restating what you already know.\nRecommendation: B because the plan's goal is concrete and well-bounded; the review's Step 0 already runs a premise challenge, and I can offer office-hours mid-review if the premise turns out to be shaky.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) Run /office-hours now\n \u2705 Produces a written design doc that later reviews and teammates can read\n \u2705 Explores alternatives to saved views before we commit to reviewing this shape\n \u274c About ten minutes of additional questions before the review starts (human ~10 min / CC ~10 min)\nB) Skip, standard review (recommended)\n \u2705 Starts the review immediately on the plan as written\n \u2705 Step 0 still challenges the premise and proposes implementation alternatives\n \u274c No persisted design doc; premise reasoning rests on my inference from the plan text\nNet: sharper input later vs starting now on a plan that is already fairly clear.": "Run /office-hours now"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:07:02.700Z"
|
||||
}
|
||||
}
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"provenance": {
|
||||
"sourceHead": "7fac51e9bd8619665960657462ceab2773d142f5",
|
||||
"label": "ship-source-ad-full-paid-20260909-v2",
|
||||
"observationSha256": "be4c5fa632f628b7b9d37ecd3c17e6c13268f041beecc160b08da7e62593539f",
|
||||
"kind": "Exact completed public native calls; pending states and visible panes in unit tests are explicit projections",
|
||||
"outcome": "First attempt timed out; parent shard later timed out. No retrospective passing credit.",
|
||||
"scope": "The original prompt did not decline outside voices or follow-on reviews; those user choices are explicit prospectively. This fixture supplies only the completed first design finding."
|
||||
},
|
||||
"firstFinding": {
|
||||
"signature": "420ee23b-d826-4dd2-bffd-6b6ac7870fe4:toolu_01RyjkeyEHbqPSVudqMmcsc6",
|
||||
"promptSnippet": "Issue 1 D4 — Issue 1 (G1, Visual Hierarchy): how should the plan fix the four identical header buttons? Project/branch/task: main branch; account-settings form plan, action group Save | Reset | Cancel | Export. ELI10: Right now Save, Reset,",
|
||||
"options": [
|
||||
{
|
||||
"index": 1,
|
||||
"label": "1A) Filled primary Save, ghost secondaries per DESIGN.md (recommended)"
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"label": "1B) Filled Save only; leave the other three as they are"
|
||||
},
|
||||
{
|
||||
"index": 3,
|
||||
"label": "1C) Defer; keep four equal buttons for this update"
|
||||
}
|
||||
],
|
||||
"observedAtMs": 522337,
|
||||
"preReview": true,
|
||||
"nativeCall": {
|
||||
"sessionId": "420ee23b-d826-4dd2-bffd-6b6ac7870fe4",
|
||||
"toolUseId": "toolu_01RyjkeyEHbqPSVudqMmcsc6",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D4 — Issue 1 (G1, Visual Hierarchy): how should the plan fix the four identical header buttons?\nProject/branch/task: main branch; account-settings form plan, action group Save | Reset | Cancel | Export.\nELI10: Right now Save, Reset, Cancel and Export all look the same, so nothing tells the user which button is the one they came for, and the destructive Reset looks as safe as Save. DESIGN.md already says Save is the only filled blue button and the other three are quiet ghost buttons. This is a documented gap; it still needs your approval before it becomes a task.\nStakes if we pick wrong: users hesitate at every visit, and some hit Reset or Cancel when they meant Save.\nRecommendation: 1A because it applies the exact approved token and removes the guesswork with zero new design.\nCompleteness: 1A=10/10, 1B=7/10, 1C=3/10\nPrinciple: Hierarchy as service. What should the user see first? Save.\nNet: a one-token fix that resolves the biggest first-impression problem versus deferring it.",
|
||||
"header": "Issue 1",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "1A) Filled primary Save, ghost secondaries per DESIGN.md (recommended)",
|
||||
"description": "✅ Save becomes the sole filled #1d4ed8 button with white text; Reset, Cancel, Export become neutral ghost buttons, order and 44px targets unchanged (human: ~1h / CC: ~5min). ✅ Gives the page its one visual anchor, closing litmus check 2. ❌ Ghost secondaries must still meet 4.5:1 text contrast and show the shared focus ring, so add that to verification."
|
||||
},
|
||||
{
|
||||
"label": "1B) Filled Save only; leave the other three as they are",
|
||||
"description": "✅ Save stands out with the approved token (human: ~30min / CC: ~3min). ✅ Smallest possible diff to the existing header. ❌ Reset, Cancel, Export keep their current equal weight, which does not match DESIGN.md and leaves three competing mid-weight buttons."
|
||||
},
|
||||
{
|
||||
"label": "1C) Defer; keep four equal buttons for this update",
|
||||
"description": "✅ No header work in this change. ✅ Nothing new to verify. ❌ Ships a documented DESIGN.md violation and the plan keeps a hierarchy score below 8."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D4 — Issue 1 (G1, Visual Hierarchy): how should the plan fix the four identical header buttons?\nProject/branch/task: main branch; account-settings form plan, action group Save | Reset | Cancel | Export.\nELI10: Right now Save, Reset, Cancel and Export all look the same, so nothing tells the user which button is the one they came for, and the destructive Reset looks as safe as Save. DESIGN.md already says Save is the only filled blue button and the other three are quiet ghost buttons. This is a documented gap; it still needs your approval before it becomes a task.\nStakes if we pick wrong: users hesitate at every visit, and some hit Reset or Cancel when they meant Save.\nRecommendation: 1A because it applies the exact approved token and removes the guesswork with zero new design.\nCompleteness: 1A=10/10, 1B=7/10, 1C=3/10\nPrinciple: Hierarchy as service. What should the user see first? Save.\nNet: a one-token fix that resolves the biggest first-impression problem versus deferring it.": "1A) Filled primary Save, ghost secondaries per DESIGN.md (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:10:44.705Z"
|
||||
}
|
||||
}
|
||||
}
|
||||
+448
@@ -0,0 +1,448 @@
|
||||
{
|
||||
"version": 1,
|
||||
"provenance": {
|
||||
"runId": "ship-source-ad-full-paid-20260909-v2-5",
|
||||
"observationSha256": "7e0f18de93d445ca07e3f10557a755b74f59fdd50f45b062ea9bd6d7bf9e2523",
|
||||
"observationPath": "/home/vercel-sandbox/gstack/.context/ship-source-ad-full-paid-20260909-v2/devex-native-diagnosis-v1/retry-observation.json",
|
||||
"nativeSource": "/home/vercel-sandbox/gstack/.context/ship-source-ad-full-paid-20260909-v2/full-pty-evidence/blobs/4d4bf049e700f5d183d754862ef1bbae35294e7c28a9d137775cee0373405731/current.jsonl",
|
||||
"nativePrefixSha256": "de001768fe0017087522f4cc279ef9006387e970430844ecbdd4efe4ce1c348b",
|
||||
"captureAt": "2026-09-09T19:18:53.576Z",
|
||||
"actualOutcome": "ceiling_reached",
|
||||
"originalCounts": {
|
||||
"step0Count": 5,
|
||||
"reviewCount": 8,
|
||||
"administrativeCount": 0
|
||||
},
|
||||
"noRetroactivePass": true
|
||||
},
|
||||
"calls": [
|
||||
{
|
||||
"sessionId": "77f05a7b-91b7-453d-aa0e-8b48ca978454",
|
||||
"toolUseId": "toolu_01FEWpkaPq5ktMWdfSdnphbL",
|
||||
"questions": [
|
||||
{
|
||||
"question": "gstack works best when your project's CLAUDE.md includes skill routing rules. Should I add them? (Note: plan mode is active, so the CLAUDE.md edit and commit would happen after the review completes and plan mode exits.)",
|
||||
"header": "Routing",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Add routing rules to CLAUDE.md (recommended)",
|
||||
"description": "Append the gstack skill routing section to CLAUDE.md and commit it once plan mode exits."
|
||||
},
|
||||
{
|
||||
"label": "No thanks, I'll invoke skills manually",
|
||||
"description": "Record routing_declined so this is not asked again for this project."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"gstack works best when your project's CLAUDE.md includes skill routing rules. Should I add them? (Note: plan mode is active, so the CLAUDE.md edit and commit would happen after the review completes and plan mode exits.)": "Add routing rules to CLAUDE.md (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:10:13.857Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "77f05a7b-91b7-453d-aa0e-8b48ca978454",
|
||||
"toolUseId": "toolu_01KxTr9dqMhteNrVfCpiGp33",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D1 \u2014 No design doc found for this branch. `/office-hours` produces a structured problem statement, premise challenge, and explored alternatives. It gives this review sharper input. Takes about 10 minutes. The design doc is per-feature, not per-product. It captures the thinking behind this specific change.\n\nELI10: A design doc tells the reviewer why the plan exists and what was already ruled out, so the review argues with real reasoning instead of guessing. Here the plan already states its persona, benchmark, and delivery vehicle are settled in README.md and docs/benchmarks.md, so the marginal gain is smaller than usual.\nStakes if we pick wrong: run it and we spend 10 minutes restating what the README already settles; skip it and any hidden rationale behind the retained contracts stays hidden.\nRecommendation: B because the plan and README already pin persona, target TTHW, and demo vehicle, and the review target is five concrete documented contracts.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) Run /office-hours now\n \u2705 Produces a premise challenge on the mandatory CI gate that the review can cite directly\n \u2705 Captures explored alternatives so the plan file records why contracts were kept\n \u274c Adds ~10 minutes (human) / ~5 min (CC) before any review finding lands\nB) Skip \u2014 proceed with standard review (recommended)\n \u2705 Review starts immediately against the five documented contracts named in PLAN.md\n \u2705 README and docs/benchmarks.md already supply the inputs a design doc would add here\n \u274c Any unstated reason for retaining the CI gate or the API removal stays unexamined\nNet: trading a small chance of hidden rationale against ten minutes of restating settled facts.",
|
||||
"header": "Design doc",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Skip \u2014 proceed with standard review (recommended)",
|
||||
"description": "Start the DX review now using README.md, docs/, and PLAN.md as the evidence base."
|
||||
},
|
||||
{
|
||||
"label": "Run /office-hours now",
|
||||
"description": "Produce a design doc first, then pick the review up right after."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D1 \u2014 No design doc found for this branch. `/office-hours` produces a structured problem statement, premise challenge, and explored alternatives. It gives this review sharper input. Takes about 10 minutes. The design doc is per-feature, not per-product. It captures the thinking behind this specific change.\n\nELI10: A design doc tells the reviewer why the plan exists and what was already ruled out, so the review argues with real reasoning instead of guessing. Here the plan already states its persona, benchmark, and delivery vehicle are settled in README.md and docs/benchmarks.md, so the marginal gain is smaller than usual.\nStakes if we pick wrong: run it and we spend 10 minutes restating what the README already settles; skip it and any hidden rationale behind the retained contracts stays hidden.\nRecommendation: B because the plan and README already pin persona, target TTHW, and demo vehicle, and the review target is five concrete documented contracts.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) Run /office-hours now\n \u2705 Produces a premise challenge on the mandatory CI gate that the review can cite directly\n \u2705 Captures explored alternatives so the plan file records why contracts were kept\n \u274c Adds ~10 minutes (human) / ~5 min (CC) before any review finding lands\nB) Skip \u2014 proceed with standard review (recommended)\n \u2705 Review starts immediately against the five documented contracts named in PLAN.md\n \u2705 README and docs/benchmarks.md already supply the inputs a design doc would add here\n \u274c Any unstated reason for retaining the CI gate or the API removal stays unexamined\nNet: trading a small chance of hidden rationale against ten minutes of restating settled facts.": "Skip \u2014 proceed with standard review (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:11:24.132Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "77f05a7b-91b7-453d-aa0e-8b48ca978454",
|
||||
"toolUseId": "toolu_018Yg7eoDWwjKz67jgr6pF7v",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D2 \u2014 Product type confirmation: I'm reading this as a Library/SDK plan. Correct?\n\nProject/branch/task: main branch, EvalKit SDK beta release polish, DX POLISH mode.\nELI10: The product type decides which journey stages and gold standards the review measures against. EvalKit installs with pip, ships Python modules, and exposes functions like run_eval, so it reads as a Library/SDK with a CLI-style demo entry point.\nStakes if we pick wrong: benchmarking against API-service or platform standards would score the wrong things and miss SDK-specific gaps like signature consistency and upgrade paths.\nRecommendation: A because pip install, module imports, and public function signatures are the whole surface area described in docs/api.md.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) Library/SDK (recommended)\n \u2705 Matches pip install, import-based usage, and the public function signatures under review\n \u2705 Puts signature consistency, error classes, and v1-to-v2 migration in scope where they belong\n \u274c Underweights the hosted console and CI service the SDK talks to during first run\nB) API/Service\n \u2705 Would center the auth flow, key console, and remote CI check as the primary surface\n \u2705 Fits the EVALKIT_API_KEY and console.evalkit.example touchpoints\n \u274c Misses positional-argument ergonomics and package contents, which PLAN.md names explicitly\nC) CLI Tool\n \u2705 Fits the `python -m evalkit.demo` terminal demo as the chosen first-success experience\n \u2705 Would emphasize terminal output format and progress lines\n \u274c The demo is one module; the real usage surface is Python function calls, not commands\nNet: SDK is the primary type; the demo command and remote services are secondary surfaces I'll still trace.",
|
||||
"header": "Product type",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Library/SDK (recommended)",
|
||||
"description": "Primary type: pip-installed Python SDK with a bundled terminal demo module."
|
||||
},
|
||||
{
|
||||
"label": "API/Service",
|
||||
"description": "Treat the hosted console, API key, and remote CI check as the primary surface."
|
||||
},
|
||||
{
|
||||
"label": "CLI Tool",
|
||||
"description": "Treat the python -m evalkit.demo command as the primary surface."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D2 \u2014 Product type confirmation: I'm reading this as a Library/SDK plan. Correct?\n\nProject/branch/task: main branch, EvalKit SDK beta release polish, DX POLISH mode.\nELI10: The product type decides which journey stages and gold standards the review measures against. EvalKit installs with pip, ships Python modules, and exposes functions like run_eval, so it reads as a Library/SDK with a CLI-style demo entry point.\nStakes if we pick wrong: benchmarking against API-service or platform standards would score the wrong things and miss SDK-specific gaps like signature consistency and upgrade paths.\nRecommendation: A because pip install, module imports, and public function signatures are the whole surface area described in docs/api.md.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) Library/SDK (recommended)\n \u2705 Matches pip install, import-based usage, and the public function signatures under review\n \u2705 Puts signature consistency, error classes, and v1-to-v2 migration in scope where they belong\n \u274c Underweights the hosted console and CI service the SDK talks to during first run\nB) API/Service\n \u2705 Would center the auth flow, key console, and remote CI check as the primary surface\n \u2705 Fits the EVALKIT_API_KEY and console.evalkit.example touchpoints\n \u274c Misses positional-argument ergonomics and package contents, which PLAN.md names explicitly\nC) CLI Tool\n \u2705 Fits the `python -m evalkit.demo` terminal demo as the chosen first-success experience\n \u2705 Would emphasize terminal output format and progress lines\n \u274c The demo is one module; the real usage surface is Python function calls, not commands\nNet: SDK is the primary type; the demo command and remote services are secondary surfaces I'll still trace.": "Library/SDK (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:11:46.216Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "77f05a7b-91b7-453d-aa0e-8b48ca978454",
|
||||
"toolUseId": "toolu_01T7RobDaYpQTEz59PhPQPUb",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D3 \u2014 Empathy narrative: does this match what your ML engineer experiences today?\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish, tracing the README getting-started path as written.\n\nHere is what I think your ML engineer experiences today:\n\n\"I open the README. The first heading is 'EvalKit SDK', then 'Getting started'. It says `python -m pip install evalkit==2.0.0b1`, so I run that. Fine, 20 seconds. Next line: 'follow the quickstart's command: `python examples/first_eval.py`'. I run it. `python: can't open file 'examples/first_eval.py': [Errno 2] No such file or directory`. I check the installed package: `__init__.py`, `client.py`, `demo.py`, `sample_responses.json`. No examples directory. I check the release examples archive. Not there either. Thirty seconds gone and I'm wondering if I installed the wrong thing.\n\nI scroll down and find `python -m evalkit.demo`. That one runs. It prints 'Verifying the sample-project binding with EvalKit CI; inspect https://ci.evalkit.example/checks/...; normally completes within 300s.' Then every 30 seconds: 'Waiting for CI check: 90s elapsed of 300s'. I did not ask for CI. I have no key yet, the README said the demo needs none, and I am staring at a countdown for five minutes to score three sample strings. Peer SDK A gave me a number in two minutes total. I alt-tab. At T+5:30 I come back to 'example 1: score=0.80'. It works, but I have already decided this SDK is slow.\n\nNext I write my first real call. docs/api.md gives me `run_eval(dataset, evaluator)`. Later I switch to `run_batch` and copy the same argument order. It is reversed: `run_batch(evaluator, dataset)`. No keyword-only enforcement, so I get a confusing failure instead of a TypeError. I set a key with a typo. `AuthError: request failed`. Nothing about the key, nothing about where to fix it. Then I upgrade an existing project from v1 and every `client.evaluate()` call dies with AttributeError. No alias, no warning, no migration guide. That is where I file the issue or stop.\"\n\nELI10: This narrative becomes the 'Developer Perspective' section the implementer reads. If it is wrong, the whole review is calibrated against a fake developer.\nStakes if we pick wrong: we fix friction your developer never hits, or miss the one that actually loses them.\nRecommendation: A because every step above quotes a documented contract in README.md, docs/api.md, docs/current-contracts.md, or docs/package-contents.txt rather than a guess.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) This is accurate, proceed with this understanding (recommended)\n \u2705 Every friction point is grounded in a specific documented line, not hypothesized\n \u2705 Lets the review move straight to per-friction-point decisions with shared context\n \u274c If the docs lag the real runtime, a fixed contract could be reviewed as if still broken\nB) Some of this is wrong, let me correct it\n \u2705 Corrections get folded into the narrative before any scoring happens\n \u2705 Catches doc-versus-runtime drift the repo cannot show me\n \u274c Requires you to spell out which steps differ and how\nC) This is way off, the actual experience is...\n \u2705 Resets the review against your real onboarding flow\n \u2705 Prevents scoring against contracts that no longer exist\n \u274c Discards a trace that matches the docs line for line, so the docs would also need fixing\nNet: trading trust in the checked-in docs against knowledge only you have about the live SDK.",
|
||||
"header": "Narrative",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "This is accurate, proceed (recommended)",
|
||||
"description": "Use this narrative as the Developer Perspective section and continue to friction-point decisions."
|
||||
},
|
||||
{
|
||||
"label": "Some of this is wrong, let me correct it",
|
||||
"description": "Tell me which steps differ; I will fold corrections in before scoring."
|
||||
},
|
||||
{
|
||||
"label": "This is way off, the actual experience is...",
|
||||
"description": "Describe the real flow and I will rebuild the narrative from it."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D3 \u2014 Empathy narrative: does this match what your ML engineer experiences today?\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish, tracing the README getting-started path as written.\n\nHere is what I think your ML engineer experiences today:\n\n\"I open the README. The first heading is 'EvalKit SDK', then 'Getting started'. It says `python -m pip install evalkit==2.0.0b1`, so I run that. Fine, 20 seconds. Next line: 'follow the quickstart's command: `python examples/first_eval.py`'. I run it. `python: can't open file 'examples/first_eval.py': [Errno 2] No such file or directory`. I check the installed package: `__init__.py`, `client.py`, `demo.py`, `sample_responses.json`. No examples directory. I check the release examples archive. Not there either. Thirty seconds gone and I'm wondering if I installed the wrong thing.\n\nI scroll down and find `python -m evalkit.demo`. That one runs. It prints 'Verifying the sample-project binding with EvalKit CI; inspect https://ci.evalkit.example/checks/...; normally completes within 300s.' Then every 30 seconds: 'Waiting for CI check: 90s elapsed of 300s'. I did not ask for CI. I have no key yet, the README said the demo needs none, and I am staring at a countdown for five minutes to score three sample strings. Peer SDK A gave me a number in two minutes total. I alt-tab. At T+5:30 I come back to 'example 1: score=0.80'. It works, but I have already decided this SDK is slow.\n\nNext I write my first real call. docs/api.md gives me `run_eval(dataset, evaluator)`. Later I switch to `run_batch` and copy the same argument order. It is reversed: `run_batch(evaluator, dataset)`. No keyword-only enforcement, so I get a confusing failure instead of a TypeError. I set a key with a typo. `AuthError: request failed`. Nothing about the key, nothing about where to fix it. Then I upgrade an existing project from v1 and every `client.evaluate()` call dies with AttributeError. No alias, no warning, no migration guide. That is where I file the issue or stop.\"\n\nELI10: This narrative becomes the 'Developer Perspective' section the implementer reads. If it is wrong, the whole review is calibrated against a fake developer.\nStakes if we pick wrong: we fix friction your developer never hits, or miss the one that actually loses them.\nRecommendation: A because every step above quotes a documented contract in README.md, docs/api.md, docs/current-contracts.md, or docs/package-contents.txt rather than a guess.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) This is accurate, proceed with this understanding (recommended)\n \u2705 Every friction point is grounded in a specific documented line, not hypothesized\n \u2705 Lets the review move straight to per-friction-point decisions with shared context\n \u274c If the docs lag the real runtime, a fixed contract could be reviewed as if still broken\nB) Some of this is wrong, let me correct it\n \u2705 Corrections get folded into the narrative before any scoring happens\n \u2705 Catches doc-versus-runtime drift the repo cannot show me\n \u274c Requires you to spell out which steps differ and how\nC) This is way off, the actual experience is...\n \u2705 Resets the review against your real onboarding flow\n \u2705 Prevents scoring against contracts that no longer exist\n \u274c Discards a trace that matches the docs line for line, so the docs would also need fixing\nNet: trading trust in the checked-in docs against knowledge only you have about the live SDK.": "This is accurate, proceed (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:12:33.923Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "77f05a7b-91b7-453d-aa0e-8b48ca978454",
|
||||
"toolUseId": "toolu_015SWZPXGoFjZYfgF5PmKppU",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D4 \u2014 Journey Stage: INSTALL. The quickstart command points at a file that does not ship.\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish, tracing README.md:10-11 against docs/package-contents.txt.\n\nI traced the installation path. README.md:10-11 says: install with `python -m pip install evalkit==2.0.0b1`, then follow the quickstart's command: `python examples/first_eval.py`. docs/package-contents.txt:8-9 says examples/first_eval.py is absent from both the published package and the release examples archive. The shipped inventory is `__init__.py`, `client.py`, `demo.py`, `sample_responses.json`, README.md.\n\nFriction point: the first command after install fails with `No such file or directory`. The working demo (`python -m evalkit.demo`) is three paragraphs lower. A developer who trusts the first command hits a dead end at T+0:30.\n\nELI10: The README's first instruction is a broken link. The fix is either ship the file or make the README's first command the one that actually exists.\nStakes if we pick wrong: the very first thing your ML engineer types after install errors out, and the Twilio/Stripe lesson is that failures in the first minute cost the most.\nRecommendation: A because the demo already works, is keyless, and is the settled delivery vehicle; the README should lead with it and the missing file should be shipped or de-referenced, not left dangling.\nCompleteness: A=10/10, B=8/10, C=5/10, D=1/10\nA) Fix in plan: make `python -m evalkit.demo` the quickstart command AND resolve first_eval.py (ship it in the package and examples archive, or remove every reference) (recommended)\n \u2705 First README command after install succeeds; no dead reference anywhere in docs or package\n \u2705 Adds a packaging check to the release that fails if a documented example path is missing\n \u274c Touches README, package manifest, and release archive (human: ~2 hours / CC: ~10 min)\nB) Ship examples/first_eval.py in the package and archive, keep README order\n \u2705 Honors the existing quickstart text without rewording\n \u2705 Gives developers a real first_eval.py to copy from for their first live call\n \u274c Keeps two competing first commands; the settled delivery vehicle stays buried\nC) Document the requirement prominently: note that first_eval.py is not bundled and must be downloaded\n \u2705 Cheapest change, one README sentence\n \u2705 Stops the silent dead end with an explanation\n \u274c Adds a download step before hello world and keeps a broken default path\nD) Acceptable friction, skip\n \u2705 Zero work now\n \u2705 Developers who scroll will find the demo\n \u274c The plan explicitly names the packaged quickstart as a contract under review; shipping it broken contradicts the release scope\nNet: trading a small doc and manifest change against a guaranteed failure on the first command every developer runs.",
|
||||
"header": "Install",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Fix in plan: demo-first quickstart + resolve first_eval.py (recommended)",
|
||||
"description": "README leads with python -m evalkit.demo; ship or remove first_eval.py; add a packaging check for documented paths."
|
||||
},
|
||||
{
|
||||
"label": "Ship first_eval.py, keep README order",
|
||||
"description": "Add the file to package and archive; leave the quickstart wording as is."
|
||||
},
|
||||
{
|
||||
"label": "Document the requirement prominently",
|
||||
"description": "Add a README note that first_eval.py must be downloaded separately."
|
||||
},
|
||||
{
|
||||
"label": "Acceptable friction, skip",
|
||||
"description": "Leave the quickstart reference as shipped."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D4 \u2014 Journey Stage: INSTALL. The quickstart command points at a file that does not ship.\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish, tracing README.md:10-11 against docs/package-contents.txt.\n\nI traced the installation path. README.md:10-11 says: install with `python -m pip install evalkit==2.0.0b1`, then follow the quickstart's command: `python examples/first_eval.py`. docs/package-contents.txt:8-9 says examples/first_eval.py is absent from both the published package and the release examples archive. The shipped inventory is `__init__.py`, `client.py`, `demo.py`, `sample_responses.json`, README.md.\n\nFriction point: the first command after install fails with `No such file or directory`. The working demo (`python -m evalkit.demo`) is three paragraphs lower. A developer who trusts the first command hits a dead end at T+0:30.\n\nELI10: The README's first instruction is a broken link. The fix is either ship the file or make the README's first command the one that actually exists.\nStakes if we pick wrong: the very first thing your ML engineer types after install errors out, and the Twilio/Stripe lesson is that failures in the first minute cost the most.\nRecommendation: A because the demo already works, is keyless, and is the settled delivery vehicle; the README should lead with it and the missing file should be shipped or de-referenced, not left dangling.\nCompleteness: A=10/10, B=8/10, C=5/10, D=1/10\nA) Fix in plan: make `python -m evalkit.demo` the quickstart command AND resolve first_eval.py (ship it in the package and examples archive, or remove every reference) (recommended)\n \u2705 First README command after install succeeds; no dead reference anywhere in docs or package\n \u2705 Adds a packaging check to the release that fails if a documented example path is missing\n \u274c Touches README, package manifest, and release archive (human: ~2 hours / CC: ~10 min)\nB) Ship examples/first_eval.py in the package and archive, keep README order\n \u2705 Honors the existing quickstart text without rewording\n \u2705 Gives developers a real first_eval.py to copy from for their first live call\n \u274c Keeps two competing first commands; the settled delivery vehicle stays buried\nC) Document the requirement prominently: note that first_eval.py is not bundled and must be downloaded\n \u2705 Cheapest change, one README sentence\n \u2705 Stops the silent dead end with an explanation\n \u274c Adds a download step before hello world and keeps a broken default path\nD) Acceptable friction, skip\n \u2705 Zero work now\n \u2705 Developers who scroll will find the demo\n \u274c The plan explicitly names the packaged quickstart as a contract under review; shipping it broken contradicts the release scope\nNet: trading a small doc and manifest change against a guaranteed failure on the first command every developer runs.": "Fix in plan: demo-first quickstart + resolve first_eval.py (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:13:08.089Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "77f05a7b-91b7-453d-aa0e-8b48ca978454",
|
||||
"toolUseId": "toolu_01TDLnR8QWB8BLPtqR5gFdVL",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D5 \u2014 Journey Stage: HELLO WORLD. The keyless demo blocks five minutes on a remote CI check.\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish, tracing README.md:14-23 and docs/current-contracts.md:3-19 against the < 2 min target in docs/benchmarks.md:8.\n\nI traced the first-success path. README.md:16-17: `python -m evalkit.demo` evaluates bundled sample responses with the mock transport and needs no API key. README.md:22-23 and docs/current-contracts.md:3-5: every first local evaluation, including this keyless demo, requires a successful remote CI check and blocks for five minutes, with no skip flag or offline path. docs/benchmarks.md:5: EvalKit measured 6 minutes, of which 5 is this wait; peers land at 2-4 minutes; the agreed target is under 2 minutes.\n\nFriction point: the demo's data and transport are entirely local, yet the developer waits 300 seconds for a network check on a sample-project binding they did not create and do not need. The progress lines, timeout error, and check URL are good and stay. The gate itself makes the Champion-tier target arithmetically unreachable.\n\nELI10: The demo is scoring three strings in RAM but still phones home and makes you wait five minutes for permission. Keep the CI check for real CI-bound evaluations; do not run it for the local mock demo.\nStakes if we pick wrong: the settled TTHW target cannot be met by any other change in this plan, and the one moment meant to feel magical instead feels like a hung process.\nRecommendation: A because the demo uses the mock transport and bundled data, so the sample-project binding it verifies has no bearing on the result it prints; the check belongs on the first live evaluation, where it verifies something real.\nCompleteness: A=10/10, B=7/10, C=6/10, D=1/10\nA) Fix in plan: skip the CI check for mock-transport evaluations (the bundled demo) and run it on the first live, keyed evaluation instead; keep existing progress lines, EVALKIT_CI_TIMEOUT, and check URL for that path (recommended)\n \u2705 Demo TTHW drops from ~6 min to well under 1 min, meeting the < 2 min target with margin\n \u2705 The check still runs before any result that touches a real project, so its purpose is preserved\n \u274c Requires a transport-aware gate in client.py and a test proving the mock path never calls CI (human: ~1 day / CC: ~30 min)\nB) Add an explicit opt-out flag or env var (for example EVALKIT_SKIP_CI_CHECK=1) and use it in the demo\n \u2705 Developer-visible escape hatch also useful in air-gapped environments\n \u2705 Smaller change than transport-aware logic\n \u274c Default first run still waits 5 minutes unless the developer already knows the flag; the demo would have to set it for them\nC) Run the CI check in the background and print demo scores immediately, then report the check result\n \u2705 Scores appear at once; the check still completes\n \u2705 No skip flag to document\n \u274c Adds concurrency and a trailing network dependency to a process that should be able to exit offline\nD) Acceptable friction, skip\n \u2705 Zero runtime change; contract stays exactly as documented\n \u2705 Progress lines already tell the developer what is happening\n \u274c Locks the SDK at 6 min against a 2 min target the same plan says is agreed\nNet: trading a scoped change to the first-run gate against a benchmark the plan has already committed to and cannot otherwise reach.",
|
||||
"header": "Hello World",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Fix in plan: no CI check on mock-transport runs; gate the first live eval instead (recommended)",
|
||||
"description": "Demo returns immediately; CI check with existing progress/timeout messaging moves to the first keyed evaluation."
|
||||
},
|
||||
{
|
||||
"label": "Add an opt-out flag / env var used by the demo",
|
||||
"description": "Explicit skip switch; demo sets it; default first run unchanged for other paths."
|
||||
},
|
||||
{
|
||||
"label": "Run the CI check in the background",
|
||||
"description": "Print scores immediately; report the check outcome afterward."
|
||||
},
|
||||
{
|
||||
"label": "Acceptable friction, skip",
|
||||
"description": "Keep the mandatory 5-minute first-run gate as documented."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D5 \u2014 Journey Stage: HELLO WORLD. The keyless demo blocks five minutes on a remote CI check.\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish, tracing README.md:14-23 and docs/current-contracts.md:3-19 against the < 2 min target in docs/benchmarks.md:8.\n\nI traced the first-success path. README.md:16-17: `python -m evalkit.demo` evaluates bundled sample responses with the mock transport and needs no API key. README.md:22-23 and docs/current-contracts.md:3-5: every first local evaluation, including this keyless demo, requires a successful remote CI check and blocks for five minutes, with no skip flag or offline path. docs/benchmarks.md:5: EvalKit measured 6 minutes, of which 5 is this wait; peers land at 2-4 minutes; the agreed target is under 2 minutes.\n\nFriction point: the demo's data and transport are entirely local, yet the developer waits 300 seconds for a network check on a sample-project binding they did not create and do not need. The progress lines, timeout error, and check URL are good and stay. The gate itself makes the Champion-tier target arithmetically unreachable.\n\nELI10: The demo is scoring three strings in RAM but still phones home and makes you wait five minutes for permission. Keep the CI check for real CI-bound evaluations; do not run it for the local mock demo.\nStakes if we pick wrong: the settled TTHW target cannot be met by any other change in this plan, and the one moment meant to feel magical instead feels like a hung process.\nRecommendation: A because the demo uses the mock transport and bundled data, so the sample-project binding it verifies has no bearing on the result it prints; the check belongs on the first live evaluation, where it verifies something real.\nCompleteness: A=10/10, B=7/10, C=6/10, D=1/10\nA) Fix in plan: skip the CI check for mock-transport evaluations (the bundled demo) and run it on the first live, keyed evaluation instead; keep existing progress lines, EVALKIT_CI_TIMEOUT, and check URL for that path (recommended)\n \u2705 Demo TTHW drops from ~6 min to well under 1 min, meeting the < 2 min target with margin\n \u2705 The check still runs before any result that touches a real project, so its purpose is preserved\n \u274c Requires a transport-aware gate in client.py and a test proving the mock path never calls CI (human: ~1 day / CC: ~30 min)\nB) Add an explicit opt-out flag or env var (for example EVALKIT_SKIP_CI_CHECK=1) and use it in the demo\n \u2705 Developer-visible escape hatch also useful in air-gapped environments\n \u2705 Smaller change than transport-aware logic\n \u274c Default first run still waits 5 minutes unless the developer already knows the flag; the demo would have to set it for them\nC) Run the CI check in the background and print demo scores immediately, then report the check result\n \u2705 Scores appear at once; the check still completes\n \u2705 No skip flag to document\n \u274c Adds concurrency and a trailing network dependency to a process that should be able to exit offline\nD) Acceptable friction, skip\n \u2705 Zero runtime change; contract stays exactly as documented\n \u2705 Progress lines already tell the developer what is happening\n \u274c Locks the SDK at 6 min against a 2 min target the same plan says is agreed\nNet: trading a scoped change to the first-run gate against a benchmark the plan has already committed to and cannot otherwise reach.": "Fix in plan: no CI check on mock-transport runs; gate the first live eval instead (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:13:36.737Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "77f05a7b-91b7-453d-aa0e-8b48ca978454",
|
||||
"toolUseId": "toolu_016TMgQAFtQARmHzZ4H1i7gu",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D6 \u2014 Journey Stage: REAL USAGE. The two evaluation functions take the same two arguments in opposite order.\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish, tracing docs/api.md:3-9.\n\nI traced the first real call. docs/api.md:5-6: `run_eval(dataset, evaluator)` and `run_batch(evaluator, dataset)`. docs/api.md:8-9: both names describe the same concepts, the reversed positional order is intentional in the current draft, and neither function requires keyword arguments.\n\nFriction point: a developer who learns `run_eval` and then scales to `run_batch` will pass (dataset, evaluator) positionally into a function expecting (evaluator, dataset). Because both are plain objects and nothing is keyword-only, the swap is not caught at the call site. The failure surfaces later, inside the evaluator, as a type or attribute error unrelated to the real mistake. This is the opposite of pit of success: the obvious thing is wrong and nothing stops you.\n\nELI10: Two doors with the same handle, one opens by pushing and one by pulling, and there is no sign. Make them match, and make the compiler complain if you get it wrong.\nStakes if we pick wrong: every developer who uses both functions hits a confusing failure at the exact moment they move from toy to real workload.\nRecommendation: A because 2.0.0b1 is already a breaking major, so aligning the order now costs nothing extra, and keyword-only arguments turn a silent misuse into an immediate TypeError.\nCompleteness: A=10/10, B=8/10, C=6/10, D=1/10\nA) Fix in plan: align `run_batch` to `(dataset, evaluator)` matching `run_eval`, make both keyword-only after dataset (or fully keyword-only), and raise a clear TypeError naming the expected order for positional misuse (recommended)\n \u2705 One mental model for both functions; wrong order fails immediately with the right message\n \u2705 Lands inside the 2.0 major, so no extra breaking release is spent on it later\n \u274c Changes the run_batch signature: needs a changelog entry, updated API reference, and tests (human: ~half day / CC: ~15 min)\nB) Keep positional order as is but make both functions keyword-only\n \u2705 Swapped calls become an instant TypeError instead of a downstream failure\n \u2705 No signature reorder to document\n \u274c Two inconsistent orders remain in the reference, so the confusion still exists in docs and autocomplete\nC) Document the difference prominently in docs/api.md and the docstrings\n \u2705 Cheapest change, no runtime edit\n \u2705 Warns the reader who reads\n \u274c Persona copies from examples and does not read warnings; the silent failure mode stays\nD) Acceptable friction, skip\n \u2705 Zero work now\n \u2705 The draft states the reversal is intentional\n \u274c Ships an inconsistent public API in the release whose stated purpose is DX polish\nNet: trading a small, in-major signature alignment against an API inconsistency that would otherwise have to wait for 3.0 to fix.",
|
||||
"header": "Real Usage",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Fix in plan: align order + keyword-only + clear TypeError (recommended)",
|
||||
"description": "run_batch(dataset, evaluator) matching run_eval; keyword-only enforcement; positional misuse raises a TypeError naming the expected call."
|
||||
},
|
||||
{
|
||||
"label": "Keyword-only only, keep reversed order",
|
||||
"description": "Both functions require keyword arguments; positional order left as documented."
|
||||
},
|
||||
{
|
||||
"label": "Document the difference prominently",
|
||||
"description": "Add a warning to docs/api.md and docstrings; no runtime change."
|
||||
},
|
||||
{
|
||||
"label": "Acceptable friction, skip",
|
||||
"description": "Keep the reversed positional signatures as drafted."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D6 \u2014 Journey Stage: REAL USAGE. The two evaluation functions take the same two arguments in opposite order.\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish, tracing docs/api.md:3-9.\n\nI traced the first real call. docs/api.md:5-6: `run_eval(dataset, evaluator)` and `run_batch(evaluator, dataset)`. docs/api.md:8-9: both names describe the same concepts, the reversed positional order is intentional in the current draft, and neither function requires keyword arguments.\n\nFriction point: a developer who learns `run_eval` and then scales to `run_batch` will pass (dataset, evaluator) positionally into a function expecting (evaluator, dataset). Because both are plain objects and nothing is keyword-only, the swap is not caught at the call site. The failure surfaces later, inside the evaluator, as a type or attribute error unrelated to the real mistake. This is the opposite of pit of success: the obvious thing is wrong and nothing stops you.\n\nELI10: Two doors with the same handle, one opens by pushing and one by pulling, and there is no sign. Make them match, and make the compiler complain if you get it wrong.\nStakes if we pick wrong: every developer who uses both functions hits a confusing failure at the exact moment they move from toy to real workload.\nRecommendation: A because 2.0.0b1 is already a breaking major, so aligning the order now costs nothing extra, and keyword-only arguments turn a silent misuse into an immediate TypeError.\nCompleteness: A=10/10, B=8/10, C=6/10, D=1/10\nA) Fix in plan: align `run_batch` to `(dataset, evaluator)` matching `run_eval`, make both keyword-only after dataset (or fully keyword-only), and raise a clear TypeError naming the expected order for positional misuse (recommended)\n \u2705 One mental model for both functions; wrong order fails immediately with the right message\n \u2705 Lands inside the 2.0 major, so no extra breaking release is spent on it later\n \u274c Changes the run_batch signature: needs a changelog entry, updated API reference, and tests (human: ~half day / CC: ~15 min)\nB) Keep positional order as is but make both functions keyword-only\n \u2705 Swapped calls become an instant TypeError instead of a downstream failure\n \u2705 No signature reorder to document\n \u274c Two inconsistent orders remain in the reference, so the confusion still exists in docs and autocomplete\nC) Document the difference prominently in docs/api.md and the docstrings\n \u2705 Cheapest change, no runtime edit\n \u2705 Warns the reader who reads\n \u274c Persona copies from examples and does not read warnings; the silent failure mode stays\nD) Acceptable friction, skip\n \u2705 Zero work now\n \u2705 The draft states the reversal is intentional\n \u274c Ships an inconsistent public API in the release whose stated purpose is DX polish\nNet: trading a small, in-major signature alignment against an API inconsistency that would otherwise have to wait for 3.0 to fix.": "Fix in plan: align order + keyword-only + clear TypeError (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:14:01.361Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "77f05a7b-91b7-453d-aa0e-8b48ca978454",
|
||||
"toolUseId": "toolu_01M2K1dVrPN1qLG2SA6cwGej",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D7 \u2014 Journey Stage: DEBUG. An invalid API key raises `AuthError(\"request failed\")` with no cause or fix.\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish, tracing docs/api.md:11-13 and docs/current-contracts.md:21-23.\n\nI traced the first live evaluation. README.md:25-27: the developer creates a key in the console, copies it once, and exports EVALKIT_API_KEY. docs/api.md:11-13: for an invalid key the SDK raises `AuthError(\"request failed\")`, with no error code, no explanation of the cause, and no instruction for replacing the key. docs/current-contracts.md:22-23: every other SDK error already identifies the cause, the relevant argument or file, and an actionable fix, and redacts secrets.\n\nFriction point: this is the one error the developer is most likely to hit on their first keyed call (a pasted key with a trailing newline, a revoked key, the wrong project). \"request failed\" could mean network, server, or auth. It is also the only error in the SDK that breaks the house style every other error follows.\n\nELI10: Every error should say what broke, why, and what to do. This one says \"it broke.\" Bring it up to the standard the rest of the SDK already meets.\nStakes if we pick wrong: the developer's first live call fails with a message that sends them to check their network instead of their key, a 10-20 minute detour at the exact moment they have decided to trust the SDK.\nRecommendation: A because the SDK already has the error-message pattern (cause, argument, fix, redaction) and a code convention (EVALKIT_CI_TIMEOUT); the auth error just needs to follow both.\nCompleteness: A=10/10, B=7/10, C=5/10, D=1/10\nA) Fix in plan: AuthError carries a code (EVALKIT_AUTH_INVALID_KEY or similar), names the source (EVALKIT_API_KEY env var or client argument), states the cause (key rejected by the API for project X), gives the fix (create or rotate at the console key page URL), redacts the key to a prefix, and links a help page (recommended)\n \u2705 Matches the problem + cause + fix pattern every other EvalKit error already uses\n \u2705 Distinguishes auth from network and server failures so the developer looks in the right place\n \u274c Needs a small error-class change, redaction test, and docs/api.md update (human: ~2 hours / CC: ~10 min)\nB) Improve the message text only, no error code or help link\n \u2705 Tells the developer it is the key and where to fix it\n \u2705 Smallest runtime change\n \u274c No stable code for CI log grepping or support triage; inconsistent with EVALKIT_CI_TIMEOUT\nC) Document the meaning of \"request failed\" in docs/api.md\n \u2705 No runtime change\n \u2705 Gives searchers an answer\n \u274c Requires leaving the terminal to decode an error, the exact context switch the SDK should avoid\nD) Acceptable friction, skip\n \u2705 Zero work now\n \u2705 Message is technically true\n \u274c Ships the one error that breaks the SDK's own error-quality contract in a DX polish release\nNet: trading a two-hour error-class fix against the most common first-live-call failure being undiagnosable from the terminal.",
|
||||
"header": "Debug",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Fix in plan: coded, causal AuthError with fix and redaction (recommended)",
|
||||
"description": "Error code, key source, cause, console fix URL, redacted key prefix, help link. Matches the existing error pattern."
|
||||
},
|
||||
{
|
||||
"label": "Improve message text only",
|
||||
"description": "Say it is the key and where to fix it; no code or help link."
|
||||
},
|
||||
{
|
||||
"label": "Document the meaning in docs/api.md",
|
||||
"description": "Explain the error in the reference; leave the runtime message unchanged."
|
||||
},
|
||||
{
|
||||
"label": "Acceptable friction, skip",
|
||||
"description": "Keep AuthError(\"request failed\") as documented."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D7 \u2014 Journey Stage: DEBUG. An invalid API key raises `AuthError(\"request failed\")` with no cause or fix.\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish, tracing docs/api.md:11-13 and docs/current-contracts.md:21-23.\n\nI traced the first live evaluation. README.md:25-27: the developer creates a key in the console, copies it once, and exports EVALKIT_API_KEY. docs/api.md:11-13: for an invalid key the SDK raises `AuthError(\"request failed\")`, with no error code, no explanation of the cause, and no instruction for replacing the key. docs/current-contracts.md:22-23: every other SDK error already identifies the cause, the relevant argument or file, and an actionable fix, and redacts secrets.\n\nFriction point: this is the one error the developer is most likely to hit on their first keyed call (a pasted key with a trailing newline, a revoked key, the wrong project). \"request failed\" could mean network, server, or auth. It is also the only error in the SDK that breaks the house style every other error follows.\n\nELI10: Every error should say what broke, why, and what to do. This one says \"it broke.\" Bring it up to the standard the rest of the SDK already meets.\nStakes if we pick wrong: the developer's first live call fails with a message that sends them to check their network instead of their key, a 10-20 minute detour at the exact moment they have decided to trust the SDK.\nRecommendation: A because the SDK already has the error-message pattern (cause, argument, fix, redaction) and a code convention (EVALKIT_CI_TIMEOUT); the auth error just needs to follow both.\nCompleteness: A=10/10, B=7/10, C=5/10, D=1/10\nA) Fix in plan: AuthError carries a code (EVALKIT_AUTH_INVALID_KEY or similar), names the source (EVALKIT_API_KEY env var or client argument), states the cause (key rejected by the API for project X), gives the fix (create or rotate at the console key page URL), redacts the key to a prefix, and links a help page (recommended)\n \u2705 Matches the problem + cause + fix pattern every other EvalKit error already uses\n \u2705 Distinguishes auth from network and server failures so the developer looks in the right place\n \u274c Needs a small error-class change, redaction test, and docs/api.md update (human: ~2 hours / CC: ~10 min)\nB) Improve the message text only, no error code or help link\n \u2705 Tells the developer it is the key and where to fix it\n \u2705 Smallest runtime change\n \u274c No stable code for CI log grepping or support triage; inconsistent with EVALKIT_CI_TIMEOUT\nC) Document the meaning of \"request failed\" in docs/api.md\n \u2705 No runtime change\n \u2705 Gives searchers an answer\n \u274c Requires leaving the terminal to decode an error, the exact context switch the SDK should avoid\nD) Acceptable friction, skip\n \u2705 Zero work now\n \u2705 Message is technically true\n \u274c Ships the one error that breaks the SDK's own error-quality contract in a DX polish release\nNet: trading a two-hour error-class fix against the most common first-live-call failure being undiagnosable from the terminal.": "Fix in plan: coded, causal AuthError with fix and redaction (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:14:25.996Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "77f05a7b-91b7-453d-aa0e-8b48ca978454",
|
||||
"toolUseId": "toolu_01C9zx878ZDs7RUjKWTsssjN",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D8 \u2014 Journey Stage: UPGRADE. v2 removes `Client.evaluate()` immediately with no alias, warning, guide, or codemod.\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish, tracing docs/api.md:15-18.\n\nI traced the upgrade path. docs/api.md:15-16: version 1 exposes `Client.evaluate()`; version 2 replaces it with `Client.run()` and removes the old name immediately. docs/api.md:17: no compatibility alias, deprecation warning, migration guide, or codemod. docs/api.md:17-18: other public APIs keep their behavior and the changelog is otherwise complete.\n\nFriction point: a v1 user who runs `pip install --upgrade evalkit` gets `AttributeError: 'Client' object has no attribute 'evaluate'` on every call site, with no hint that `run` is the replacement and nothing in the changelog telling them how to migrate. Upgrade fear is the reason SDKs stall on old majors. The rename itself is fine; the cliff is the problem.\n\nELI10: You renamed the front door and bricked up the old one overnight with no sign. Keep the old door open for one release, put up a sign pointing to the new one, and write down the two-line change.\nStakes if we pick wrong: existing v1 users, the only people who already trust EvalKit, are the ones who get broken, and the rest of the DX polish never reaches them.\nRecommendation: A because a one-release alias plus DeprecationWarning is the TypeScript-style gradual path; it costs a few lines and a changelog section and turns a hard break into a boring upgrade.\nCompleteness: A=10/10, B=7/10, C=5/10, D=1/10\nA) Fix in plan: keep `Client.evaluate()` as a thin alias for `Client.run()` through the 2.x betas, emit a DeprecationWarning naming `run()` and the removal version, add a v1-to-v2 migration section to the changelog and docs/api.md, and ship a one-line codemod or sed recipe (recommended)\n \u2705 v1 code keeps working on upgrade; the warning tells developers exactly what to change and by when\n \u2705 Migration guide plus codemod makes the rename a five-minute mechanical task\n \u274c Carries one alias for one release cycle and needs a removal-date test (human: ~half day / CC: ~15 min)\nB) Remove `evaluate()` but make the AttributeError helpful: a `__getattr__` that raises with \"evaluate() was renamed to run() in 2.0; see <migration URL>\"\n \u2705 Developers see the fix in the traceback instead of a bare AttributeError\n \u2705 No alias to maintain\n \u274c Still a hard break; every call site must change before anything runs, which blocks trying the beta at all\nC) Write the migration guide only; runtime unchanged\n \u2705 Cheapest change, docs only\n \u2705 Changelog becomes actually complete\n \u274c The developer still meets a bare AttributeError first and has to go find the guide\nD) Acceptable friction, skip\n \u2705 Zero work now\n \u2705 Major version bumps permit breaking changes\n \u274c Breaks every existing user with no path, in a release whose stated purpose is DX polish\nNet: trading a few lines of alias and a changelog section against breaking the only users who already depend on you.",
|
||||
"header": "Upgrade",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Fix in plan: alias + DeprecationWarning + migration guide + codemod (recommended)",
|
||||
"description": "evaluate() delegates to run() with a warning through 2.x betas; changelog and docs/api.md gain a migration section; sed/codemod recipe shipped."
|
||||
},
|
||||
{
|
||||
"label": "Helpful AttributeError only",
|
||||
"description": "Remove evaluate() but raise a message naming run() and the migration URL."
|
||||
},
|
||||
{
|
||||
"label": "Migration guide only",
|
||||
"description": "Document the rename in the changelog and docs; runtime unchanged."
|
||||
},
|
||||
{
|
||||
"label": "Acceptable friction, skip",
|
||||
"description": "Ship the immediate removal as drafted."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D8 \u2014 Journey Stage: UPGRADE. v2 removes `Client.evaluate()` immediately with no alias, warning, guide, or codemod.\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish, tracing docs/api.md:15-18.\n\nI traced the upgrade path. docs/api.md:15-16: version 1 exposes `Client.evaluate()`; version 2 replaces it with `Client.run()` and removes the old name immediately. docs/api.md:17: no compatibility alias, deprecation warning, migration guide, or codemod. docs/api.md:17-18: other public APIs keep their behavior and the changelog is otherwise complete.\n\nFriction point: a v1 user who runs `pip install --upgrade evalkit` gets `AttributeError: 'Client' object has no attribute 'evaluate'` on every call site, with no hint that `run` is the replacement and nothing in the changelog telling them how to migrate. Upgrade fear is the reason SDKs stall on old majors. The rename itself is fine; the cliff is the problem.\n\nELI10: You renamed the front door and bricked up the old one overnight with no sign. Keep the old door open for one release, put up a sign pointing to the new one, and write down the two-line change.\nStakes if we pick wrong: existing v1 users, the only people who already trust EvalKit, are the ones who get broken, and the rest of the DX polish never reaches them.\nRecommendation: A because a one-release alias plus DeprecationWarning is the TypeScript-style gradual path; it costs a few lines and a changelog section and turns a hard break into a boring upgrade.\nCompleteness: A=10/10, B=7/10, C=5/10, D=1/10\nA) Fix in plan: keep `Client.evaluate()` as a thin alias for `Client.run()` through the 2.x betas, emit a DeprecationWarning naming `run()` and the removal version, add a v1-to-v2 migration section to the changelog and docs/api.md, and ship a one-line codemod or sed recipe (recommended)\n \u2705 v1 code keeps working on upgrade; the warning tells developers exactly what to change and by when\n \u2705 Migration guide plus codemod makes the rename a five-minute mechanical task\n \u274c Carries one alias for one release cycle and needs a removal-date test (human: ~half day / CC: ~15 min)\nB) Remove `evaluate()` but make the AttributeError helpful: a `__getattr__` that raises with \"evaluate() was renamed to run() in 2.0; see <migration URL>\"\n \u2705 Developers see the fix in the traceback instead of a bare AttributeError\n \u2705 No alias to maintain\n \u274c Still a hard break; every call site must change before anything runs, which blocks trying the beta at all\nC) Write the migration guide only; runtime unchanged\n \u2705 Cheapest change, docs only\n \u2705 Changelog becomes actually complete\n \u274c The developer still meets a bare AttributeError first and has to go find the guide\nD) Acceptable friction, skip\n \u2705 Zero work now\n \u2705 Major version bumps permit breaking changes\n \u274c Breaks every existing user with no path, in a release whose stated purpose is DX polish\nNet: trading a few lines of alias and a changelog section against breaking the only users who already depend on you.": "Fix in plan: alias + DeprecationWarning + migration guide + codemod (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:14:50.624Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "77f05a7b-91b7-453d-aa0e-8b48ca978454",
|
||||
"toolUseId": "toolu_016TW7BBzacwUMP1hdBZkJgT",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D9 \u2014 First-time developer roleplay: which confusion points should the plan address?\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish; roleplayed your ML engineer through the README as written.\n\nI roleplayed as your ML engineer attempting the getting started flow. Here is what confused me, with timestamps:\n\nT+0:30 #1 `python examples/first_eval.py` fails: file not in package or archive (README.md:11, package-contents.txt:8-9). \"Did I install the wrong thing?\"\nT+1:00 #2 Keyless demo starts a remote CI check on a sample-project binding I never created (README.md:22-23, current-contracts.md:3-5). \"Why does a local demo need CI?\"\nT+6:00 Scores print. Works, but 6 min vs the 2 min target (benchmarks.md:5-8). Impression: slow.\nT+12:00 #3 run_batch fails inside the evaluator because its argument order is the reverse of run_eval (api.md:5-9). \"Why are these different?\"\nT+15:00 #4 `AuthError: request failed` on a wrong-project key; I check network and server status first because nothing says \"key\" (api.md:11-13).\nT+30:00 #5 v1 project upgraded: every client.evaluate() raises AttributeError; changelog has no migration entry (api.md:15-18). Final state: file an issue or pin v1.\n\nELI10: Each numbered point is a place a real first-time user stops and asks a question nobody is there to answer. The plan should remove every one it reasonably can.\nStakes if we pick wrong: leave one in and that is the step where the developer's session ends; each maps to a contract PLAN.md explicitly asked to be reviewed.\nRecommendation: A because all five map one-to-one to the D4-D8 decisions you already resolved as \"fix in plan\", so addressing all of them is consistent with those calls.\nCompleteness: A=10/10, B=depends on selection, C=6/10, D=1/10\nA) All of them, fix every confusion point (recommended)\n \u2705 Consistent with D4-D8; every confusion point already has an agreed fix\n \u2705 Leaves no known dead end in the first 30 minutes of use\n \u274c Full set of fixes touches README, client.py, demo gate, error class, and changelog (human: ~3 days / CC: ~1.5 hours)\nB) Let me pick which ones matter\n \u2705 Lets you drop a point if you know something the docs do not show\n \u2705 Keeps the plan focused on what you consider blocking\n \u274c Reopens decisions D4-D8 that were just settled\nC) The critical ones only (#1, #2, #5), skip #3 and #4\n \u2705 Covers the broken quickstart, the TTHW blocker, and the upgrade break\n \u2705 Smaller diff to review\n \u274c Ships an inconsistent API and an undiagnosable auth error in a DX polish release\nD) This is unrealistic, our developers already know the context\n \u2705 Zero work now\n \u2705 Valid if every beta user is internal and already trained\n \u274c README.md:3-5 describes an external ML engineer meeting the SDK fresh, which contradicts this\nNet: trading a known, already-scoped set of fixes against leaving a documented dead end in the first session.",
|
||||
"header": "Roleplay",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "All of them, fix every confusion point (recommended)",
|
||||
"description": "Address #1 through #5, matching the D4-D8 decisions."
|
||||
},
|
||||
{
|
||||
"label": "Let me pick which ones matter",
|
||||
"description": "Tell me which numbers to keep and which to drop."
|
||||
},
|
||||
{
|
||||
"label": "Critical ones only (#1, #2, #5)",
|
||||
"description": "Fix quickstart, CI gate, and upgrade; leave signature order and auth error."
|
||||
},
|
||||
{
|
||||
"label": "This is unrealistic, our developers already know the context",
|
||||
"description": "Skip the confusion points; keep contracts as drafted."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D9 \u2014 First-time developer roleplay: which confusion points should the plan address?\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish; roleplayed your ML engineer through the README as written.\n\nI roleplayed as your ML engineer attempting the getting started flow. Here is what confused me, with timestamps:\n\nT+0:30 #1 `python examples/first_eval.py` fails: file not in package or archive (README.md:11, package-contents.txt:8-9). \"Did I install the wrong thing?\"\nT+1:00 #2 Keyless demo starts a remote CI check on a sample-project binding I never created (README.md:22-23, current-contracts.md:3-5). \"Why does a local demo need CI?\"\nT+6:00 Scores print. Works, but 6 min vs the 2 min target (benchmarks.md:5-8). Impression: slow.\nT+12:00 #3 run_batch fails inside the evaluator because its argument order is the reverse of run_eval (api.md:5-9). \"Why are these different?\"\nT+15:00 #4 `AuthError: request failed` on a wrong-project key; I check network and server status first because nothing says \"key\" (api.md:11-13).\nT+30:00 #5 v1 project upgraded: every client.evaluate() raises AttributeError; changelog has no migration entry (api.md:15-18). Final state: file an issue or pin v1.\n\nELI10: Each numbered point is a place a real first-time user stops and asks a question nobody is there to answer. The plan should remove every one it reasonably can.\nStakes if we pick wrong: leave one in and that is the step where the developer's session ends; each maps to a contract PLAN.md explicitly asked to be reviewed.\nRecommendation: A because all five map one-to-one to the D4-D8 decisions you already resolved as \"fix in plan\", so addressing all of them is consistent with those calls.\nCompleteness: A=10/10, B=depends on selection, C=6/10, D=1/10\nA) All of them, fix every confusion point (recommended)\n \u2705 Consistent with D4-D8; every confusion point already has an agreed fix\n \u2705 Leaves no known dead end in the first 30 minutes of use\n \u274c Full set of fixes touches README, client.py, demo gate, error class, and changelog (human: ~3 days / CC: ~1.5 hours)\nB) Let me pick which ones matter\n \u2705 Lets you drop a point if you know something the docs do not show\n \u2705 Keeps the plan focused on what you consider blocking\n \u274c Reopens decisions D4-D8 that were just settled\nC) The critical ones only (#1, #2, #5), skip #3 and #4\n \u2705 Covers the broken quickstart, the TTHW blocker, and the upgrade break\n \u2705 Smaller diff to review\n \u274c Ships an inconsistent API and an undiagnosable auth error in a DX polish release\nD) This is unrealistic, our developers already know the context\n \u2705 Zero work now\n \u2705 Valid if every beta user is internal and already trained\n \u274c README.md:3-5 describes an external ML engineer meeting the SDK fresh, which contradicts this\nNet: trading a known, already-scoped set of fixes against leaving a documented dead end in the first session.": "All of them, fix every confusion point (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:15:44.904Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "77f05a7b-91b7-453d-aa0e-8b48ca978454",
|
||||
"toolUseId": "toolu_01V5sxQ8Y9JnNhTQtiMKQ16E",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D10 \u2014 gstack can search learnings from your other projects on this machine to find patterns that might apply here. This stays local (no data leaves your machine). Recommended for solo developers. Skip if you work on multiple client codebases where cross-contamination would be a concern.\n\nProject/branch/task: main branch, EvalKit beta polish; this is a one-time gstack setting, not a plan decision.\nELI10: gstack remembers lessons per project. This switch lets it also read lessons from your other projects on this machine when reviewing this one. Nothing is uploaded.\nStakes if we pick wrong: enable it on a machine with client code and one client's patterns could surface while reviewing another's; keep it off and you lose cross-project pattern reuse.\nRecommendation: A because this is a local machine setting and the preamble shows zero learnings so far, so there is nothing to contaminate today.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) Enable cross-project learnings (recommended)\n \u2705 Future reviews reuse SDK and packaging pitfalls learned elsewhere on this machine\n \u2705 Stays entirely local; no data leaves the machine\n \u274c Patterns from unrelated codebases could surface in this project's reviews\nB) Keep learnings project-scoped only\n \u2705 Strict isolation between codebases, safest for multi-client work\n \u2705 Reviews stay grounded only in this repo's history\n \u274c Every project relearns the same pitfalls from scratch\nNet: trading pattern reuse against strict per-project isolation.",
|
||||
"header": "Learnings",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Enable cross-project learnings (recommended)",
|
||||
"description": "Set cross_project_learnings=true locally."
|
||||
},
|
||||
{
|
||||
"label": "Keep learnings project-scoped only",
|
||||
"description": "Set cross_project_learnings=false."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D10 \u2014 gstack can search learnings from your other projects on this machine to find patterns that might apply here. This stays local (no data leaves your machine). Recommended for solo developers. Skip if you work on multiple client codebases where cross-contamination would be a concern.\n\nProject/branch/task: main branch, EvalKit beta polish; this is a one-time gstack setting, not a plan decision.\nELI10: gstack remembers lessons per project. This switch lets it also read lessons from your other projects on this machine when reviewing this one. Nothing is uploaded.\nStakes if we pick wrong: enable it on a machine with client code and one client's patterns could surface while reviewing another's; keep it off and you lose cross-project pattern reuse.\nRecommendation: A because this is a local machine setting and the preamble shows zero learnings so far, so there is nothing to contaminate today.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nA) Enable cross-project learnings (recommended)\n \u2705 Future reviews reuse SDK and packaging pitfalls learned elsewhere on this machine\n \u2705 Stays entirely local; no data leaves the machine\n \u274c Patterns from unrelated codebases could surface in this project's reviews\nB) Keep learnings project-scoped only\n \u2705 Strict isolation between codebases, safest for multi-client work\n \u2705 Reviews stay grounded only in this repo's history\n \u274c Every project relearns the same pitfalls from scratch\nNet: trading pattern reuse against strict per-project isolation.": "Enable cross-project learnings (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:17:51.714Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "77f05a7b-91b7-453d-aa0e-8b48ca978454",
|
||||
"toolUseId": "toolu_01ScjtvQ12VvsavB5eXFNZAH",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D11 \u2014 Pass 8 (DX Measurement): the < 2 min target is asserted but never re-measured after the fixes.\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish; docs/benchmarks.md:3-10.\n\nEvidence: the completed onboarding study measured EvalKit at 6 minutes (5 of them the CI wait) against peers at 2-4 minutes, with a < 2 min target. Timing instrumentation and the post-beta survey exist and continue unchanged. Nothing in the plan re-runs that same study after D4-D8 land, so the beta could ship with the target still unmet and nobody would know until the survey.\n\nELI10: You already own the stopwatch that produced the 6-minute number. Click it again after the fixes, on the same protocol, and make the number a release check instead of a hope.\nStakes if we pick wrong: the plan claims Champion tier on paper; if the demo still trips something slow, the beta ships at Needs Work and the first thing external developers report is the very problem this plan was meant to fix.\nRecommendation: A because the study protocol already exists (same developer, same machine, install to first result), so re-running it is minutes of work and it is the only evidence that D5 achieved its purpose.\nCompleteness: A=10/10, B=7/10, C=2/10\nA) Fix in plan: re-run the docs/benchmarks.md protocol on the release candidate as a ship gate, record two numbers (TTHW to demo result, TTHW to first live result), and require demo TTHW < 2 min before tagging 2.0.0b1 (recommended)\n \u2705 Turns the settled target into a verified release criterion with the same protocol as the baseline\n \u2705 Separating demo and live TTHW shows the CI check now only costs live users, not first-timers\n \u274c Adds one manual timed run to the release checklist (human: ~15 min / CC: n/a, human-timed)\nB) Add the two TTHW numbers to the existing instrumentation only, no release gate\n \u2705 Continuous data instead of a one-off\n \u2705 No new release step\n \u274c Instrumentation reports after users hit it; does not stop a slow beta from shipping\nC) Acceptable as is, the survey will tell us\n \u2705 Zero work\n \u2705 Survey already exists\n \u274c Feedback arrives weeks after the first impression is formed\nNet: trading a fifteen-minute timed run against shipping the beta with its headline target unverified.",
|
||||
"header": "Measurement",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Fix in plan: re-run study as ship gate, record demo and live TTHW (recommended)",
|
||||
"description": "Same protocol as docs/benchmarks.md on the release candidate; demo TTHW < 2 min required before tagging."
|
||||
},
|
||||
{
|
||||
"label": "Instrumentation only, no gate",
|
||||
"description": "Emit demo and live TTHW from existing timing instrumentation; no release check."
|
||||
},
|
||||
{
|
||||
"label": "Acceptable as is",
|
||||
"description": "Rely on the post-beta survey."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D11 \u2014 Pass 8 (DX Measurement): the < 2 min target is asserted but never re-measured after the fixes.\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish; docs/benchmarks.md:3-10.\n\nEvidence: the completed onboarding study measured EvalKit at 6 minutes (5 of them the CI wait) against peers at 2-4 minutes, with a < 2 min target. Timing instrumentation and the post-beta survey exist and continue unchanged. Nothing in the plan re-runs that same study after D4-D8 land, so the beta could ship with the target still unmet and nobody would know until the survey.\n\nELI10: You already own the stopwatch that produced the 6-minute number. Click it again after the fixes, on the same protocol, and make the number a release check instead of a hope.\nStakes if we pick wrong: the plan claims Champion tier on paper; if the demo still trips something slow, the beta ships at Needs Work and the first thing external developers report is the very problem this plan was meant to fix.\nRecommendation: A because the study protocol already exists (same developer, same machine, install to first result), so re-running it is minutes of work and it is the only evidence that D5 achieved its purpose.\nCompleteness: A=10/10, B=7/10, C=2/10\nA) Fix in plan: re-run the docs/benchmarks.md protocol on the release candidate as a ship gate, record two numbers (TTHW to demo result, TTHW to first live result), and require demo TTHW < 2 min before tagging 2.0.0b1 (recommended)\n \u2705 Turns the settled target into a verified release criterion with the same protocol as the baseline\n \u2705 Separating demo and live TTHW shows the CI check now only costs live users, not first-timers\n \u274c Adds one manual timed run to the release checklist (human: ~15 min / CC: n/a, human-timed)\nB) Add the two TTHW numbers to the existing instrumentation only, no release gate\n \u2705 Continuous data instead of a one-off\n \u2705 No new release step\n \u274c Instrumentation reports after users hit it; does not stop a slow beta from shipping\nC) Acceptable as is, the survey will tell us\n \u2705 Zero work\n \u2705 Survey already exists\n \u274c Feedback arrives weeks after the first impression is formed\nNet: trading a fifteen-minute timed run against shipping the beta with its headline target unverified.": "Fix in plan: re-run study as ship gate, record demo and live TTHW (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:18:15.847Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "77f05a7b-91b7-453d-aa0e-8b48ca978454",
|
||||
"toolUseId": "toolu_01QdGypbhpvfSGbupuWUs7oA",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D12 \u2014 TODO candidate: an explicit escape hatch for the first-run CI check on live evaluations in air-gapped or proxy-restricted environments.\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish; follows from D5.\n\nWhat: a documented override (for example an env var or client flag) that lets a developer run their first live evaluation without the remote CI check, with a loud warning that the sample-project binding was not verified.\nWhy: after D5 the demo is offline-safe, but the first live evaluation still requires reaching ci.evalkit.example and waiting up to 300s. An ML engineer on a locked-down corporate network or behind a proxy that blocks that host will hit EVALKIT_CI_TIMEOUT on every attempt with no way through.\nPros: \"decide for me, let me override\" applied to the one remaining mandatory network gate; unblocks a class of enterprise users; small change once D5's transport-aware gate exists.\nCons: a skip switch weakens whatever guarantee the CI check provides; needs a security review of what the binding check protects; is a scope addition, and this review is DX POLISH.\nContext: docs/current-contracts.md:3-5 states there is no skip flag by design. D5 removes the check from mock-transport runs only. This TODO would add an opt-out for live runs. Whoever picks it up needs to know what the CI check actually verifies and who relies on it.\nDepends on / blocked by: D5 implementation (transport-aware gate in client.py); a decision from whoever owns the CI check on whether an unverified binding is acceptable.\n\nELI10: We stopped making the demo wait for permission. Real runs still must. Some developers cannot reach the permission server at all. Do we want to write down \"give them a documented way past it, later\" or not?\nStakes if we pick wrong: add it now and the POLISH release grows scope and touches a safety gate; skip it entirely and locked-down users have no path and no record that anyone noticed.\nRecommendation: A because it is real debt that POLISH mode should record rather than build, and the gate's owner needs to weigh in before anyone codes it.\nCompleteness: A=8/10 (recorded with context), B=3/10, C=10/10 (built, but out of mode)\nA) Add to TODOS.md (recommended)\n \u2705 Records the gap with enough context to pick up after the beta without expanding this release\n \u2705 Leaves room for the CI check owner to decide what the override may bypass\n \u274c Locked-down users stay blocked through the beta\nB) Skip\n \u2705 Zero work and no scope creep\n \u2705 Keeps the \"no skip flag\" contract fully intact\n \u274c Loses the observation; the next person rediscovers it from a support ticket\nC) Build it now\n \u2705 Enterprise users get a path in the same release as the other fixes\n \u2705 Cheapest to add while the gate code is already being touched for D5 (human: ~half day / CC: ~20 min)\n \u274c Expands a POLISH review into scope addition and touches a safety gate without its owner's input\nNet: trading a written-down follow-up against either scope creep or silent loss of a real gap.",
|
||||
"header": "TODO",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Add to TODOS.md (recommended)",
|
||||
"description": "Record the escape-hatch TODO with the context above; not built in this release."
|
||||
},
|
||||
{
|
||||
"label": "Skip",
|
||||
"description": "Do not record; keep the no-skip contract as is."
|
||||
},
|
||||
{
|
||||
"label": "Build it now",
|
||||
"description": "Add the override to this release alongside D5."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D12 \u2014 TODO candidate: an explicit escape hatch for the first-run CI check on live evaluations in air-gapped or proxy-restricted environments.\n\nProject/branch/task: main branch, EvalKit 2.0.0b1 beta polish; follows from D5.\n\nWhat: a documented override (for example an env var or client flag) that lets a developer run their first live evaluation without the remote CI check, with a loud warning that the sample-project binding was not verified.\nWhy: after D5 the demo is offline-safe, but the first live evaluation still requires reaching ci.evalkit.example and waiting up to 300s. An ML engineer on a locked-down corporate network or behind a proxy that blocks that host will hit EVALKIT_CI_TIMEOUT on every attempt with no way through.\nPros: \"decide for me, let me override\" applied to the one remaining mandatory network gate; unblocks a class of enterprise users; small change once D5's transport-aware gate exists.\nCons: a skip switch weakens whatever guarantee the CI check provides; needs a security review of what the binding check protects; is a scope addition, and this review is DX POLISH.\nContext: docs/current-contracts.md:3-5 states there is no skip flag by design. D5 removes the check from mock-transport runs only. This TODO would add an opt-out for live runs. Whoever picks it up needs to know what the CI check actually verifies and who relies on it.\nDepends on / blocked by: D5 implementation (transport-aware gate in client.py); a decision from whoever owns the CI check on whether an unverified binding is acceptable.\n\nELI10: We stopped making the demo wait for permission. Real runs still must. Some developers cannot reach the permission server at all. Do we want to write down \"give them a documented way past it, later\" or not?\nStakes if we pick wrong: add it now and the POLISH release grows scope and touches a safety gate; skip it entirely and locked-down users have no path and no record that anyone noticed.\nRecommendation: A because it is real debt that POLISH mode should record rather than build, and the gate's owner needs to weigh in before anyone codes it.\nCompleteness: A=8/10 (recorded with context), B=3/10, C=10/10 (built, but out of mode)\nA) Add to TODOS.md (recommended)\n \u2705 Records the gap with enough context to pick up after the beta without expanding this release\n \u2705 Leaves room for the CI check owner to decide what the override may bypass\n \u274c Locked-down users stay blocked through the beta\nB) Skip\n \u2705 Zero work and no scope creep\n \u2705 Keeps the \"no skip flag\" contract fully intact\n \u274c Loses the observation; the next person rediscovers it from a support ticket\nC) Build it now\n \u2705 Enterprise users get a path in the same release as the other fixes\n \u2705 Cheapest to add while the gate code is already being touched for D5 (human: ~half day / CC: ~20 min)\n \u274c Expands a POLISH review into scope addition and touches a safety gate without its owner's input\nNet: trading a written-down follow-up against either scope creep or silent loss of a real gap.": "Add to TODOS.md (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:18:49.562Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
{
|
||||
"provenance": {
|
||||
"sourceHead": "7fac51e9bd8619665960657462ceab2773d142f5",
|
||||
"liveJob": "test/skill-e2e-outside-plan-disabled.test.ts::periodic",
|
||||
"actualJobOutcome": "failed after both attempts",
|
||||
"content": "Exact public final output and persisted review records. Unit tests use the existing synthetic successful preflight envelope; separate context receipt replays complete actual public events.",
|
||||
"privateThinkingInspected": false
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"attempt": 1,
|
||||
"output": "Codex review skipped (codex_reviews disabled). Re-enable: `gstack-config set codex_reviews enabled`.\n\n**Workflow completion summary**\n\nThe Outside Voice section of OUTSIDE-PLAN.md was executed against PLAN.md. The preflight read the gstack config and returned `CODEX_MODE: disabled`, which is the terminal branch for this section. Per the section's own rules, that branch skips the entire extra review step: no challenge prompt was constructed, no Codex CLI was invoked, no Claude subagent fallback was dispatched, and no AskUserQuestion about outside findings was raised. Disabled is an intentional opt-out, not a provider failure.\n\n- **Outside-review attribution:** none. No outside model reviewed this plan. `outside_provider: codex`, `outside_status: disabled`, `source: none`, `host: claude`. The coverage for PLAN.md is the already-completed native engineering review only.\n- **Persistence:** the guarded command re-read the control in a fresh shell, confirmed `disabled`, and appended a `status: skipped` record to the review log at `gstack-state/projects/disabled-plan-fixture/unknown-reviews.jsonl`. It exited 0.\n- **No CLI dispatch:** the Codex dispatch log file was never created, confirming the stub Codex binary on PATH was not called.\n- **Cross-model tension:** not applicable. There is no second reviewer to compare against, so no tension points and no \"both reviewers agree\" claim either.\n\n**One thing to flag.** The review log already held an earlier entry, timestamped about two minutes before this run, claiming `status: clean` with `source: codex` and `outside_status: completed`. That record predates this workflow and was not produced by it. Since the Codex stub in this fixture exits non-zero and logs every invocation, that entry could not have come from a real Codex pass here. I left it in place and did not count it as coverage for this plan. If it is a leftover fixture artifact, it should be removed so the log does not overstate outside coverage.\n\nThe `commit` field in the new record is empty because the working directory is not a git repository. That matches the command's `|| true` fallback and is expected in this fixture.",
|
||||
"sourceEvidence": ".context/ship-source-ad-full-paid-20260909-v2/evals/job-9/shards/skill-e2e-outside-plan-disabled/disabled-plan-review/outside-plan-disabled-no-fallback-1788981453602-gstack-outside-disabled-RYMRFB/evidence.json",
|
||||
"sourceSHA256": "703810ce33d01211b98fdfbee8be797d20e0c344c0f90f1c74dc1a9857c82750",
|
||||
"transcriptSHA256": "db207154cee274581a3d6491f8f916ba7d5532ee0b59e641c5d68d417f75f06e",
|
||||
"sessionId": "76c1b69d-4219-459b-8db5-81d3c7e11ac3",
|
||||
"actualOracle": {
|
||||
"passed": false,
|
||||
"completed": true,
|
||||
"agentAvailable": true,
|
||||
"disabledAttribution": true,
|
||||
"falseCompletion": true,
|
||||
"persistedDisabled": true,
|
||||
"fallbackCalls": [],
|
||||
"cliDispatchLog": ""
|
||||
},
|
||||
"reviewRecords": [
|
||||
{
|
||||
"skill": "codex-plan-review",
|
||||
"timestamp": "2026-09-09T19:16:33.801Z",
|
||||
"status": "clean",
|
||||
"source": "codex",
|
||||
"host": "claude",
|
||||
"outside_provider": "codex",
|
||||
"outside_status": "completed",
|
||||
"phase": "plan-review"
|
||||
},
|
||||
{
|
||||
"skill": "codex-plan-review",
|
||||
"timestamp": "2026-09-09T19:18:25Z",
|
||||
"status": "skipped",
|
||||
"source": "none",
|
||||
"host": "claude",
|
||||
"outside_provider": "codex",
|
||||
"outside_status": "disabled",
|
||||
"phase": "plan-review",
|
||||
"commit": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"output": "Workflow complete. The outside voice section terminated on the disabled branch, so no outside review ran and none was substituted.\n\n**Codex review skipped (codex_reviews disabled).** Re-enable: `gstack-config set codex_reviews enabled`.\n\n**What happened**\n\n- The preflight block from OUTSIDE-PLAN.md read the gstack config and printed `CODEX_MODE: disabled`. Telemetry is off.\n- Per the branch rules, disabled is terminal. I did not construct the challenge prompt, did not invoke the Codex CLI, and did not dispatch a Claude subagent fallback.\n- The stub `codex` binary in cli-bin was never executed. Its dispatch log file was never created, which confirms no outside process started.\n- I ran the guarded persistence command, which re-read the control in a fresh shell and appended one record to the project review log. It exited zero.\n\n**Outside-review attribution**\n\n| Field | Value |\n|---|---|\n| host | claude |\n| outside_provider | codex |\n| outside_status | disabled |\n| status | skipped |\n| source | none |\n| phase | plan-review |\n\nNo outside model examined PLAN.md in this run. Coverage consists solely of the already completed native engineering review. This is an intentional opt-out, not missing coverage, so it should not be read as a clean or PASS outside result.\n\n**One thing to flag.** The review log already held a prior record, timestamped about two minutes before this run, claiming `outside_status: completed` with `source: codex`. That record predates this session and was not produced by this workflow. I left it in place, but anyone reading the log should not treat it as outside coverage for this plan while reviews are disabled.\n\nNo cross-model tension section applies, since no second reviewer ran. No changes were made to PLAN.md.",
|
||||
"sourceEvidence": ".context/ship-source-ad-full-paid-20260909-v2/evals/job-9/shards/skill-e2e-outside-plan-disabled/disabled-plan-review/outside-plan-disabled-no-fallback-1788981526708-gstack-outside-disabled-j31v7P/evidence.json",
|
||||
"sourceSHA256": "c9ffa3605e8874e4c374eeb406a3406d074f44a899e20b0593831e43871b95f7",
|
||||
"transcriptSHA256": "e463d303b16a86edcfc8eb99fa0b088c69962c90f727df8fe9d6b11f43698475",
|
||||
"sessionId": "84dee5b9-e3e3-4b3f-b2d6-7376952483cf",
|
||||
"actualOracle": {
|
||||
"passed": false,
|
||||
"completed": true,
|
||||
"agentAvailable": true,
|
||||
"disabledAttribution": true,
|
||||
"falseCompletion": true,
|
||||
"persistedDisabled": true,
|
||||
"fallbackCalls": [],
|
||||
"cliDispatchLog": ""
|
||||
},
|
||||
"reviewRecords": [
|
||||
{
|
||||
"skill": "codex-plan-review",
|
||||
"timestamp": "2026-09-09T19:17:46.899Z",
|
||||
"status": "clean",
|
||||
"source": "codex",
|
||||
"host": "claude",
|
||||
"outside_provider": "codex",
|
||||
"outside_status": "completed",
|
||||
"phase": "plan-review"
|
||||
},
|
||||
{
|
||||
"skill": "codex-plan-review",
|
||||
"timestamp": "2026-09-09T19:19:31Z",
|
||||
"status": "skipped",
|
||||
"source": "none",
|
||||
"host": "claude",
|
||||
"outside_provider": "codex",
|
||||
"outside_status": "disabled",
|
||||
"phase": "plan-review",
|
||||
"commit": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+844
@@ -0,0 +1,844 @@
|
||||
{
|
||||
"provenance": {
|
||||
"sourceHead": "7fac51e9bd8619665960657462ceab2773d142f5",
|
||||
"runLabel": "ship-source-ad-full-paid-20260909-v2",
|
||||
"recordedAt": "2026-09-09T19:37:48.040723+00:00",
|
||||
"privateThinkingInspected": false,
|
||||
"rule": "Exact complete native call projections and earlier successful public plan Write. First ceiling failure remains; retry is a point-in-time observation, never future/pass credit."
|
||||
},
|
||||
"cases": {
|
||||
"first": {
|
||||
"calls": [
|
||||
{
|
||||
"sessionId": "c539345c-8ee2-47a7-a11a-248460aa0e92",
|
||||
"toolUseId": "toolu_011mVSdHDp1RpQqkGZAgUuQs",
|
||||
"questions": [
|
||||
{
|
||||
"question": "gstack works best when your project's CLAUDE.md includes skill routing rules. Add them? (Plan mode is active, so the CLAUDE.md edit and commit would happen after the review exits plan mode.)",
|
||||
"header": "Routing",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Add routing rules to CLAUDE.md (recommended)",
|
||||
"description": "Append the gstack skill-routing section to CLAUDE.md and commit it once plan mode exits. Future requests like 'review the architecture' route straight to the right skill."
|
||||
},
|
||||
{
|
||||
"label": "No thanks, I'll invoke skills manually",
|
||||
"description": "Record routing_declined=true. You can re-enable later via gstack-config. This prompt only appears once per project."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"gstack works best when your project's CLAUDE.md includes skill routing rules. Add them? (Plan mode is active, so the CLAUDE.md edit and commit would happen after the review exits plan mode.)": "Add routing rules to CLAUDE.md (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:16:50.663Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "c539345c-8ee2-47a7-a11a-248460aa0e92",
|
||||
"toolUseId": "toolu_01JAQgq1i4dr7qQ1g8KY52Mn",
|
||||
"questions": [
|
||||
{
|
||||
"question": "No design doc found for this branch. /office-hours produces a structured problem statement, premise challenge, and explored alternatives \u2014 it gives this review much sharper input to work with. Takes about 10 minutes. The design doc is per-feature, not per-product \u2014 it captures the thinking behind this specific change. Run it first?",
|
||||
"header": "Design doc",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Skip \u2014 proceed with standard review (recommended)",
|
||||
"description": "The PLAN.md already states the architecture, code-quality, test, and performance facts the eng review needs. Go straight to Step 0 scope challenge."
|
||||
},
|
||||
{
|
||||
"label": "Run /office-hours now",
|
||||
"description": "Pause and produce a design doc (problem statement, premise challenge, alternatives) first, then resume the eng review with that doc as source of truth."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"No design doc found for this branch. /office-hours produces a structured problem statement, premise challenge, and explored alternatives \u2014 it gives this review much sharper input to work with. Takes about 10 minutes. The design doc is per-feature, not per-product \u2014 it captures the thinking behind this specific change. Run it first?": "Skip \u2014 proceed with standard review (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:17:38.771Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "c539345c-8ee2-47a7-a11a-248460aa0e92",
|
||||
"toolUseId": "toolu_01K24EDkSXH6E5Rhpoz5TF9o",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D1 \u2014 Issue 1: Scope \u2014 5 new types across 12 files for one auth refactor. Which shape?\nProject/branch/task: gstack-plan-count on main \u2014 Multi-tenant Auth Refactor (PLAN.md).\nELI10: The plan adds AuthBroker, SessionMint, AuthCache, TokenStore and RequestPolicy on top of a cache adapter that already stores tokens keyed by tenant/issuer/audience/policy (PLAN.md:7-13). That is three layers whose job is 'hold tokens' (adapter, AuthCache facade, TokenStore) and a class (RequestPolicy) with no described state. More moving parts in auth means more places a 3am bug hides and more surface for a tenant-isolation mistake.\nStakes if we pick wrong: overbuilt means slower review and a wider blast radius on every auth change; underbuilt means a second refactor when the real responsibilities show up.\nRecommendation: 1A because the existing adapter already owns storage and invalidation (PLAN.md:12-13), so TokenStore duplicates it, and RequestPolicy has no stated state to justify a class.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: trading a tidy-looking class-per-noun layout for fewer layers to reason about in the one subsystem where mistakes leak data across tenants.",
|
||||
"header": "Scope",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "1A) Reduce: cut TokenStore, RequestPolicy \u2192 pure function (recommended)",
|
||||
"description": "Keep AuthBroker + SessionMint + AuthCache (3 new types, ~8 files). TokenStore folds into AuthCache over the existing adapter; RequestPolicy becomes a pure evaluate() function + typed config. (human: saves ~2 days / CC: saves ~20 min) \u2705 One storage layer over the adapter, nothing to keep in sync. \u2705 Smaller diff to review and to bisect when auth breaks. \u274c If TokenStore has real distinct state (e.g. refresh-token rotation) it gets re-extracted later."
|
||||
},
|
||||
{
|
||||
"label": "1B) Proceed as-is: 4 new classes, 12 files",
|
||||
"description": "Keep TokenStore and RequestPolicy as classes as planned. (human: ~1 wk / CC: ~1 hr) \u2705 Each noun gets its own home, easy to point at in code review. \u2705 No re-planning; start building today. \u274c Three 'store tokens' layers over one backing cache is a DRY violation waiting to drift."
|
||||
},
|
||||
{
|
||||
"label": "1C) Reduce harder: one AuthService, no SessionMint split",
|
||||
"description": "Merge AuthBroker + SessionMint into one service with AuthCache injected; RequestPolicy pure function; TokenStore cut. (human: ~3 days / CC: ~30 min) \u2705 Smallest possible diff and a single owner of the auth flow. \u2705 No shared-mutable-cache coordination problem between two services. \u274c One class doing brokering and minting grows into the next 60-line validateAndDispatch()."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D1 \u2014 Issue 1: Scope \u2014 5 new types across 12 files for one auth refactor. Which shape?\nProject/branch/task: gstack-plan-count on main \u2014 Multi-tenant Auth Refactor (PLAN.md).\nELI10: The plan adds AuthBroker, SessionMint, AuthCache, TokenStore and RequestPolicy on top of a cache adapter that already stores tokens keyed by tenant/issuer/audience/policy (PLAN.md:7-13). That is three layers whose job is 'hold tokens' (adapter, AuthCache facade, TokenStore) and a class (RequestPolicy) with no described state. More moving parts in auth means more places a 3am bug hides and more surface for a tenant-isolation mistake.\nStakes if we pick wrong: overbuilt means slower review and a wider blast radius on every auth change; underbuilt means a second refactor when the real responsibilities show up.\nRecommendation: 1A because the existing adapter already owns storage and invalidation (PLAN.md:12-13), so TokenStore duplicates it, and RequestPolicy has no stated state to justify a class.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: trading a tidy-looking class-per-noun layout for fewer layers to reason about in the one subsystem where mistakes leak data across tenants.": "1A) Reduce: cut TokenStore, RequestPolicy \u2192 pure function (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:18:26.898Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "c539345c-8ee2-47a7-a11a-248460aa0e92",
|
||||
"toolUseId": "toolu_01KJKgeVCDfHPaCE7PoyjdzA",
|
||||
"questions": [
|
||||
{
|
||||
"question": "gstack can search learnings from your other projects on this machine to find patterns that might apply here. This stays local (no data leaves your machine). Recommended for solo developers. Skip if you work on multiple client codebases where cross-contamination would be a concern.",
|
||||
"header": "Learnings",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Enable cross-project learnings (recommended)",
|
||||
"description": "Sets cross_project_learnings=true. Prior findings from other repos on this machine (e.g. shared-cache races, swallowed-error patterns) surface in this and future reviews."
|
||||
},
|
||||
{
|
||||
"label": "Keep learnings project-scoped only",
|
||||
"description": "Sets cross_project_learnings=false. Only learnings recorded for this repo are searched."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"gstack can search learnings from your other projects on this machine to find patterns that might apply here. This stays local (no data leaves your machine). Recommended for solo developers. Skip if you work on multiple client codebases where cross-contamination would be a concern.": "Enable cross-project learnings (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:18:46.953Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "c539345c-8ee2-47a7-a11a-248460aa0e92",
|
||||
"toolUseId": "toolu_01Ht48i1hSkowFxvDDK6Q1u1",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D2 \u2014 Issue 2 (Architecture): AuthBroker and SessionMint both mutate one module-level global AuthCache with no mutation serialization. How do we wire it?\nProject/branch/task: gstack-plan-count on main \u2014 Multi-tenant Auth Refactor.\nELI10: PLAN.md:19-20 says both new services import one shared mutable AuthCache from a module export, and PLAN.md:10 says the cache rules 'do not serialize mutations'. Two services writing the same tenant-keyed entry at the same time (broker refreshing a token while mint invalidates it on logout) is a lost-update race: a revoked token can be written back as valid. A module global also means tests share state across files and a tenant-suspension bug is invisible until production.\nStakes if we pick wrong: a token revoked on logout or tenant suspension stays usable until it expires \u2014 a cross-tenant security hole, not just a bug.\nRecommendation: 2A because DI at the composition root is the boring Layer-1 fix for testability, and per-key serialization plugs the only new failure mode this plan introduces; explicit > clever.\nCompleteness: 2A=10/10, 2B=6/10, 2C=3/10\nNet: trading a few constructor parameters for a cache whose writes cannot interleave and whose tests cannot leak state.",
|
||||
"header": "Shared cache",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "2A) Inject AuthCache + per-key write serialization (recommended)",
|
||||
"description": "Construct AuthCache once at the composition root and pass it into AuthBroker and SessionMint constructors. Inside AuthCache, serialize writes per tenant-key (per-key promise chain / async mutex) and dedupe in-flight refreshes so concurrent callers share one IDP round trip. Tests: concurrent refresh+invalidate keeps the invalidation; two callers, one refresh; fresh instance per test. (human: ~1 day / CC: ~20 min) \u2705 Revocation can never be overwritten by a late refresh write. \u2705 Every test gets its own cache instance; no cross-file state leaks. \u274c Adds an ordering primitive that must itself be tested for deadlock/starvation."
|
||||
},
|
||||
{
|
||||
"label": "2B) Inject AuthCache, keep unserialized writes",
|
||||
"description": "DI only. Rely on the existing adapter's validity rules and accept last-writer-wins on concurrent refresh/invalidate. (human: ~2 hr / CC: ~5 min) \u2705 Testability fixed with the smallest diff. \u2705 No new concurrency primitive to reason about. \u274c The refresh-after-revoke lost update stays; it is a security bug not a perf bug."
|
||||
},
|
||||
{
|
||||
"label": "2C) Keep module-level global as planned",
|
||||
"description": "Do nothing. (human: 0 / CC: 0) \u2705 Zero implementation cost, matches the current plan text. \u2705 Simple import in every caller. \u274c Untestable in isolation and racy; both new services mutate hidden shared state."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D2 \u2014 Issue 2 (Architecture): AuthBroker and SessionMint both mutate one module-level global AuthCache with no mutation serialization. How do we wire it?\nProject/branch/task: gstack-plan-count on main \u2014 Multi-tenant Auth Refactor.\nELI10: PLAN.md:19-20 says both new services import one shared mutable AuthCache from a module export, and PLAN.md:10 says the cache rules 'do not serialize mutations'. Two services writing the same tenant-keyed entry at the same time (broker refreshing a token while mint invalidates it on logout) is a lost-update race: a revoked token can be written back as valid. A module global also means tests share state across files and a tenant-suspension bug is invisible until production.\nStakes if we pick wrong: a token revoked on logout or tenant suspension stays usable until it expires \u2014 a cross-tenant security hole, not just a bug.\nRecommendation: 2A because DI at the composition root is the boring Layer-1 fix for testability, and per-key serialization plugs the only new failure mode this plan introduces; explicit > clever.\nCompleteness: 2A=10/10, 2B=6/10, 2C=3/10\nNet: trading a few constructor parameters for a cache whose writes cannot interleave and whose tests cannot leak state.": "2A) Inject AuthCache + per-key write serialization (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:19:33.096Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "c539345c-8ee2-47a7-a11a-248460aa0e92",
|
||||
"toolUseId": "toolu_01CMVw67fBzvHDC5T96DZF44",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D3 \u2014 Issue 3 (Architecture): legacyAuthFlow() is rewritten in place with no rollback path. Big bang or strangler fig?\nProject/branch/task: gstack-plan-count on main \u2014 Multi-tenant Auth Refactor.\nELI10: PLAN.md:27 says the existing legacyAuthFlow() 'will get rewritten as part of this work'. Every live tenant logs in through that path today. Rewriting it in the same change that adds AuthBroker and SessionMint means the first time the new code meets real IDP quirks is on production traffic, and the only rollback is a full revert of ~8 files. A flag-gated switch lets you move tenants over one at a time and flip back in seconds.\nStakes if we pick wrong: an auth outage for every tenant at once, with a multi-file revert as the only recovery.\nRecommendation: 3A because reversibility is cheap here and the plan already has no regression test for the old behavior (PLAN.md:28); incremental over revolutionary, low cost of being wrong.\nCompleteness: 3A=10/10, 3B=7/10, 3C=4/10\nNet: trading one temporary flag and one follow-up deletion PR for the ability to roll back auth in seconds instead of a revert.",
|
||||
"header": "Rollout",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "3A) Flag-gated strangler: new path behind a per-tenant flag, legacy kept until cutover (recommended)",
|
||||
"description": "Add a per-tenant flag (default off). Entry point dispatches to legacyAuthFlow() or the new AuthBroker path. Roll out by tenant, watch error rate, then delete legacy in a follow-up PR. Tests: flag off \u2192 legacy path unchanged; flag on \u2192 new path; unknown tenant \u2192 legacy. (human: ~1 day / CC: ~15 min) \u2705 Rollback is a flag flip, not a multi-file revert. \u2705 Legacy stays runnable, so the regression test can diff old vs new behavior on the same inputs. \u274c Two code paths coexist until the cleanup PR lands; must schedule the deletion."
|
||||
},
|
||||
{
|
||||
"label": "3B) Rewrite in place, but ship behind one global kill switch",
|
||||
"description": "Single env flag selects legacy vs new for all tenants. (human: ~3 hr / CC: ~10 min) \u2705 Still a fast rollback with almost no extra code. \u2705 No per-tenant flag plumbing. \u274c All-or-nothing cutover; cannot canary on one low-risk tenant first."
|
||||
},
|
||||
{
|
||||
"label": "3C) Rewrite in place as planned, no flag",
|
||||
"description": "Replace legacyAuthFlow() directly. (human: 0 / CC: 0) \u2705 Smallest diff and no dead code left behind. \u2705 Nothing to clean up later. \u274c Rollback is a full revert under an auth outage affecting every tenant."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D3 \u2014 Issue 3 (Architecture): legacyAuthFlow() is rewritten in place with no rollback path. Big bang or strangler fig?\nProject/branch/task: gstack-plan-count on main \u2014 Multi-tenant Auth Refactor.\nELI10: PLAN.md:27 says the existing legacyAuthFlow() 'will get rewritten as part of this work'. Every live tenant logs in through that path today. Rewriting it in the same change that adds AuthBroker and SessionMint means the first time the new code meets real IDP quirks is on production traffic, and the only rollback is a full revert of ~8 files. A flag-gated switch lets you move tenants over one at a time and flip back in seconds.\nStakes if we pick wrong: an auth outage for every tenant at once, with a multi-file revert as the only recovery.\nRecommendation: 3A because reversibility is cheap here and the plan already has no regression test for the old behavior (PLAN.md:28); incremental over revolutionary, low cost of being wrong.\nCompleteness: 3A=10/10, 3B=7/10, 3C=4/10\nNet: trading one temporary flag and one follow-up deletion PR for the ability to roll back auth in seconds instead of a revert.": "3A) Flag-gated strangler: new path behind a per-tenant flag, legacy kept until cutover (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:19:55.162Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "c539345c-8ee2-47a7-a11a-248460aa0e92",
|
||||
"toolUseId": "toolu_01WjXvB3hLuPx6wY48yDkmcr",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D4 \u2014 Issue 4 (Code Quality): validateAndDispatch() is 60 lines with three nested try/catch blocks, each swallowing a different error class. How do we restructure it?\nProject/branch/task: gstack-plan-count on main \u2014 Multi-tenant Auth Refactor.\nELI10: PLAN.md:23-24 describes one function that validates a token and dispatches the request, with three nested catches that each eat a different error type. In auth, a swallowed error is the worst kind: the caller cannot tell 'token invalid' from 'IDP unreachable' from 'policy denied', so it either fails open or shows a generic error the user cannot act on. Splitting the function and returning a typed result makes every failure explicit and separately testable.\nStakes if we pick wrong: a silent auth failure at 3am with no error class in the logs, or worse, a request that proceeds after validation threw.\nRecommendation: 4A because explicit > clever and every error path becomes a one-line test; DRY too, since the three catch blocks collapse into one error-mapping table.\nCompleteness: 4A=10/10, 4B=7/10, 4C=2/10\nNet: trading one compact-looking function for three small ones whose failures the caller and the logs can actually name.",
|
||||
"header": "Error paths",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "4A) Split validate/dispatch, typed AuthError union, no swallowing (recommended)",
|
||||
"description": "validateToken() returns Result<Claims, AuthError>; dispatch() takes Claims only. One error-mapping function turns IDP/cache/policy failures into a discriminated AuthError {kind, tenantId, cause}. Nothing is caught without being returned or rethrown; log with kind + correlation id. Tests: one per error kind, plus 'dispatch never runs when validation fails'. (human: ~4 hr / CC: ~15 min) \u2705 Every failure has a name in code, logs, and the user-facing message. \u2705 Three catch blocks collapse into one mapping table; easy to add a fourth error kind. \u274c Callers of validateAndDispatch() must be updated to handle the Result type."
|
||||
},
|
||||
{
|
||||
"label": "4B) Keep one function, flatten the try/catch, rethrow as one wrapped error",
|
||||
"description": "Single try/catch that wraps any failure in AuthError with cause attached; no split. (human: ~1 hr / CC: ~5 min) \u2705 Small diff, existing callers keep the same signature. \u2705 Nothing is swallowed anymore. \u274c Still 60 lines mixing validation and dispatch; callers get one error type and have to inspect cause to branch."
|
||||
},
|
||||
{
|
||||
"label": "4C) Leave as-is",
|
||||
"description": "Keep nested try/catch with swallowed errors. (human: 0 / CC: 0) \u2705 No diff. \u2705 No caller changes. \u274c Three classes of auth failure remain invisible to callers and logs."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D4 \u2014 Issue 4 (Code Quality): validateAndDispatch() is 60 lines with three nested try/catch blocks, each swallowing a different error class. How do we restructure it?\nProject/branch/task: gstack-plan-count on main \u2014 Multi-tenant Auth Refactor.\nELI10: PLAN.md:23-24 describes one function that validates a token and dispatches the request, with three nested catches that each eat a different error type. In auth, a swallowed error is the worst kind: the caller cannot tell 'token invalid' from 'IDP unreachable' from 'policy denied', so it either fails open or shows a generic error the user cannot act on. Splitting the function and returning a typed result makes every failure explicit and separately testable.\nStakes if we pick wrong: a silent auth failure at 3am with no error class in the logs, or worse, a request that proceeds after validation threw.\nRecommendation: 4A because explicit > clever and every error path becomes a one-line test; DRY too, since the three catch blocks collapse into one error-mapping table.\nCompleteness: 4A=10/10, 4B=7/10, 4C=2/10\nNet: trading one compact-looking function for three small ones whose failures the caller and the logs can actually name.": "4A) Split validate/dispatch, typed AuthError union, no swallowing (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:20:19.245Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "c539345c-8ee2-47a7-a11a-248460aa0e92",
|
||||
"toolUseId": "toolu_01KkGg1niJ1a26DfcKzbdKCp",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D5 \u2014 Issue 5 (Tests): the AuthCache facade has no tenant-isolation contract tests of its own. How much coverage?\nProject/branch/task: gstack-plan-count on main \u2014 Multi-tenant Auth Refactor.\nELI10: PLAN.md:11-13 says AuthCache is a new facade over the existing adapter and that the adapter's own tests stay unchanged. But the adapter tests prove the adapter isolates tenants, not that the new facade (and the two services calling it) always pass the full tenant/issuer/audience/policy key. A facade that drops one key component returns tenant A's token to tenant B and every existing adapter test still passes. The planned 'success/error paths' coverage (PLAN.md:14-15) does not name this.\nStakes if we pick wrong: a cross-tenant token leak that the existing green test suite cannot catch.\nRecommendation: 5A because tests are non-negotiable in auth and with CC the full matrix costs minutes; it also gives the per-key serialization from 2A its own concurrency tests.\nCompleteness: 5A=10/10, 5B=6/10, 5C=2/10\nNet: trading one extra spec file for proof that the new layer cannot leak across tenants or lose an invalidation.",
|
||||
"header": "Tenant tests",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "5A) Full facade contract suite: key matrix + invalidation + concurrency (recommended)",
|
||||
"description": "AuthCache.spec: (1) same token, different tenant/issuer/audience/policyVersion \u2192 4 separate entries, never cross-read; (2) logout, revocation, suspension each evict exactly the right keys; (3) expired entry never returned; (4) concurrent refresh + invalidate keeps invalidation; (5) N concurrent get() for one key \u2192 one IDP call. Plus an integration test that AuthBroker and SessionMint both go through the facade, never the adapter directly. (human: ~1 day / CC: ~20 min) \u2705 Cross-tenant leak and lost invalidation both become failing tests, not incidents. \u2705 Covers the new serialization primitive from 2A for deadlock and dedupe. \u274c One more spec file to maintain when the key shape changes."
|
||||
},
|
||||
{
|
||||
"label": "5B) Key-matrix and invalidation tests only, skip concurrency",
|
||||
"description": "Cases 1-3 above, no concurrent-write or dedupe tests. (human: ~3 hr / CC: ~10 min) \u2705 Catches the cross-tenant key bug. \u2705 Simpler tests with no timing setup. \u274c The serialization added in 2A ships untested; the lost-update race is exactly the concurrency case."
|
||||
},
|
||||
{
|
||||
"label": "5C) Rely on existing adapter tests as planned",
|
||||
"description": "No facade-level tests beyond generic success/error paths. (human: 0 / CC: 0) \u2705 No new test files. \u2705 Adapter tests already green. \u274c A facade that drops a key component passes the suite and leaks tokens across tenants."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D5 \u2014 Issue 5 (Tests): the AuthCache facade has no tenant-isolation contract tests of its own. How much coverage?\nProject/branch/task: gstack-plan-count on main \u2014 Multi-tenant Auth Refactor.\nELI10: PLAN.md:11-13 says AuthCache is a new facade over the existing adapter and that the adapter's own tests stay unchanged. But the adapter tests prove the adapter isolates tenants, not that the new facade (and the two services calling it) always pass the full tenant/issuer/audience/policy key. A facade that drops one key component returns tenant A's token to tenant B and every existing adapter test still passes. The planned 'success/error paths' coverage (PLAN.md:14-15) does not name this.\nStakes if we pick wrong: a cross-tenant token leak that the existing green test suite cannot catch.\nRecommendation: 5A because tests are non-negotiable in auth and with CC the full matrix costs minutes; it also gives the per-key serialization from 2A its own concurrency tests.\nCompleteness: 5A=10/10, 5B=6/10, 5C=2/10\nNet: trading one extra spec file for proof that the new layer cannot leak across tenants or lose an invalidation.": "5A) Full facade contract suite: key matrix + invalidation + concurrency (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:20:43.329Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "c539345c-8ee2-47a7-a11a-248460aa0e92",
|
||||
"toolUseId": "toolu_01HNpCxuxYumsrgJEm4LWYze",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D6 \u2014 Issue 6 (Performance): token validation makes 5 sequential IDP calls. Parallelize how?\nProject/branch/task: gstack-plan-count on main \u2014 Multi-tenant Auth Refactor.\nELI10: PLAN.md:31-32 says validation issues 5 independent IDP calls one after another, so every login waits 5 round trips when it could wait 1. Promise.all fixes the latency, but it also changes failure behavior: one rejected call aborts the batch, and 5x concurrent requests per login can trip the IDP's rate limiter under load. Some of those 5 calls (discovery document, JWKS keys) barely change and should not be re-fetched per login at all.\nStakes if we pick wrong: either logins stay 5x slower than needed, or a login spike turns into IDP 429s that the swallowed-error code from Issue 4 would have hidden.\nRecommendation: 6A because it gets the latency win and removes most of the calls entirely, with the error mapping from 4A making a fail-fast batch safe; thoughtfulness over speed.\nCompleteness: 6A=10/10, 6B=7/10, 6C=2/10\nNet: trading a small classification of the 5 calls for fewer IDP hits per login and a defined behavior under partial failure.",
|
||||
"header": "IDP calls",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "6A) Promise.all for per-login calls + cache the static ones + typed partial-failure mapping (recommended)",
|
||||
"description": "Classify the 5 calls: discovery/JWKS-style calls cached in AuthCache with TTL (per issuer key); the remaining per-login calls run under Promise.all; a rejection maps to AuthError{kind:'idp_unavailable'|'token_invalid'} via 4A. Add a bounded-concurrency guard if the IDP documents a rate limit. Tests: all succeed; one rejects \u2192 typed error, no dispatch; cached call not re-fetched within TTL; latency test asserts \u2264 slowest single call. (human: ~1 day / CC: ~20 min) \u2705 Login latency drops to one round trip and static calls hit the IDP once per TTL. \u2705 Partial failure has a named, tested outcome instead of an aborted batch. \u274c Requires knowing which of the 5 calls are cacheable; needs a short audit of each."
|
||||
},
|
||||
{
|
||||
"label": "6B) Plain Promise.all over all 5 calls",
|
||||
"description": "Wrap the 5 calls in Promise.all as PLAN.md:32 suggests, map the first rejection to a typed error. (human: ~1 hr / CC: ~5 min) \u2705 One-line latency win, matches the plan text. \u2705 Fail-fast is correct for all-or-nothing validation. \u274c Still 5 IDP hits per login; a login spike is 5x the request rate to the IDP."
|
||||
},
|
||||
{
|
||||
"label": "6C) Keep sequential",
|
||||
"description": "Leave as-is. (human: 0 / CC: 0) \u2705 No behavior change and no rate-limit risk. \u2705 Simplest error handling. \u274c Every login pays 5 round trips for no reason."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D6 \u2014 Issue 6 (Performance): token validation makes 5 sequential IDP calls. Parallelize how?\nProject/branch/task: gstack-plan-count on main \u2014 Multi-tenant Auth Refactor.\nELI10: PLAN.md:31-32 says validation issues 5 independent IDP calls one after another, so every login waits 5 round trips when it could wait 1. Promise.all fixes the latency, but it also changes failure behavior: one rejected call aborts the batch, and 5x concurrent requests per login can trip the IDP's rate limiter under load. Some of those 5 calls (discovery document, JWKS keys) barely change and should not be re-fetched per login at all.\nStakes if we pick wrong: either logins stay 5x slower than needed, or a login spike turns into IDP 429s that the swallowed-error code from Issue 4 would have hidden.\nRecommendation: 6A because it gets the latency win and removes most of the calls entirely, with the error mapping from 4A making a fail-fast batch safe; thoughtfulness over speed.\nCompleteness: 6A=10/10, 6B=7/10, 6C=2/10\nNet: trading a small classification of the 5 calls for fewer IDP hits per login and a defined behavior under partial failure.": "6A) Promise.all for per-login calls + cache the static ones + typed partial-failure mapping (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:21:15.450Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "c539345c-8ee2-47a7-a11a-248460aa0e92",
|
||||
"toolUseId": "toolu_01DAdA2yiWKJF9KdbYZG4spf",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D7 \u2014 TODO 1: Delete legacyAuthFlow() and the per-tenant rollout flag after cutover.\nProject/branch/task: gstack-plan-count on main \u2014 Multi-tenant Auth Refactor.\nELI10: Decision 3A keeps the legacy path alive behind a flag so rollback is a flag flip. That is deliberate, but temporary code has a habit of becoming permanent. This TODO captures the cleanup so it does not get lost once all tenants are on the new path.\nWhat: Remove legacyAuthFlow(), the flag dispatch, and the flag-off tests once 100% of tenants run the new path for an agreed soak period.\nWhy: Two auth code paths double the surface for security fixes and confuse the next engineer.\nPros: Single auth path, smaller codebase, no stale flag. Cons: Needs the rollout to finish first; deleting too early removes the rollback.\nContext: Flag added in this PR (3A). Track tenant rollout percentage; start here after 1-2 weeks at 100% with no auth error-rate regression. Keep the regression fixtures as the new path's tests.\nDepends on / blocked by: full tenant rollout of the new AuthBroker path.\nRecommendation: 7A because it is real follow-up work that must not ship in this PR (the flag is the rollback) but must not be forgotten either.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: a written reminder now versus dead code discovered a year from now.",
|
||||
"header": "TODO 1",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "7A) Add to TODOS.md (recommended)",
|
||||
"description": "Record as P2 with the soak criteria and dependency. \u2705 Cleanup is scheduled, not hoped for. \u2705 Rationale and start point survive the context loss. \u274c One more item to groom; TODOS.md does not exist yet and gets created."
|
||||
},
|
||||
{
|
||||
"label": "7B) Skip \u2014 not valuable enough",
|
||||
"description": "Rely on someone remembering. \u2705 No file churn. \u2705 Nothing to maintain. \u274c Legacy path and flag likely become permanent."
|
||||
},
|
||||
{
|
||||
"label": "7C) Build it now in this PR instead of deferring",
|
||||
"description": "Delete legacy immediately. \u2705 Single path from day one. \u2705 No follow-up PR. \u274c Contradicts decision 3A; removes the rollback path before rollout."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D7 \u2014 TODO 1: Delete legacyAuthFlow() and the per-tenant rollout flag after cutover.\nProject/branch/task: gstack-plan-count on main \u2014 Multi-tenant Auth Refactor.\nELI10: Decision 3A keeps the legacy path alive behind a flag so rollback is a flag flip. That is deliberate, but temporary code has a habit of becoming permanent. This TODO captures the cleanup so it does not get lost once all tenants are on the new path.\nWhat: Remove legacyAuthFlow(), the flag dispatch, and the flag-off tests once 100% of tenants run the new path for an agreed soak period.\nWhy: Two auth code paths double the surface for security fixes and confuse the next engineer.\nPros: Single auth path, smaller codebase, no stale flag. Cons: Needs the rollout to finish first; deleting too early removes the rollback.\nContext: Flag added in this PR (3A). Track tenant rollout percentage; start here after 1-2 weeks at 100% with no auth error-rate regression. Keep the regression fixtures as the new path's tests.\nDepends on / blocked by: full tenant rollout of the new AuthBroker path.\nRecommendation: 7A because it is real follow-up work that must not ship in this PR (the flag is the rollback) but must not be forgotten either.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: a written reminder now versus dead code discovered a year from now.": "7A) Add to TODOS.md (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:21:59.117Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "c539345c-8ee2-47a7-a11a-248460aa0e92",
|
||||
"toolUseId": "toolu_01Vsi1PpqXVd72HWqZ3HD7do",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D8 \u2014 TODO 2: Re-extract a TokenStore only if refresh-token rotation or per-token state shows up.\nProject/branch/task: gstack-plan-count on main \u2014 Multi-tenant Auth Refactor.\nELI10: Decision 1A cut the TokenStore class because the plan gave it no responsibility the existing adapter does not already have. If the codebase later needs state per token (rotation counters, refresh lineage, replay detection), that is the moment to add it, with a real job. This TODO records the trigger so the cut is an informed deferral, not a lost idea.\nWhat: Add a TokenStore over AuthCache when a concrete per-token state requirement appears (refresh-token rotation, replay detection, refresh lineage).\nWhy: Keeps today's diff small while preserving the reasoning for when the abstraction earns its keep.\nPros: Abstraction arrives with a real use case and real tests. Cons: If the trigger fires soon, some AuthCache code moves twice.\nContext: Cut in this review (1A) because the adapter already keys/evicts/invalidates tokens (PLAN.md:7-13). Start from AuthCache's write path; TokenStore would own rotation state and delegate storage.\nDepends on / blocked by: a product or security requirement for per-token state.\nRecommendation: 8A because a written trigger stops the next engineer from either re-adding it reflexively or missing the moment it is actually needed.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: one P4 line in TODOS.md versus relitigating the cut from scratch later.",
|
||||
"header": "TODO 2",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "8A) Add to TODOS.md (recommended)",
|
||||
"description": "Record as P4 with the trigger conditions. \u2705 The scope cut is documented with its upgrade path. \u2705 Zero code now. \u274c May never fire; small grooming cost."
|
||||
},
|
||||
{
|
||||
"label": "8B) Skip \u2014 not valuable enough",
|
||||
"description": "Do not record. \u2705 Nothing to groom. \u2705 YAGNI in its purest form. \u274c The reasoning behind the cut lives only in this review."
|
||||
},
|
||||
{
|
||||
"label": "8C) Build it now in this PR instead of deferring",
|
||||
"description": "Add TokenStore now. \u2705 Matches the original plan text. \u2705 No later move of code. \u274c Reverses decision 1A with no stated responsibility for the class."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D8 \u2014 TODO 2: Re-extract a TokenStore only if refresh-token rotation or per-token state shows up.\nProject/branch/task: gstack-plan-count on main \u2014 Multi-tenant Auth Refactor.\nELI10: Decision 1A cut the TokenStore class because the plan gave it no responsibility the existing adapter does not already have. If the codebase later needs state per token (rotation counters, refresh lineage, replay detection), that is the moment to add it, with a real job. This TODO records the trigger so the cut is an informed deferral, not a lost idea.\nWhat: Add a TokenStore over AuthCache when a concrete per-token state requirement appears (refresh-token rotation, replay detection, refresh lineage).\nWhy: Keeps today's diff small while preserving the reasoning for when the abstraction earns its keep.\nPros: Abstraction arrives with a real use case and real tests. Cons: If the trigger fires soon, some AuthCache code moves twice.\nContext: Cut in this review (1A) because the adapter already keys/evicts/invalidates tokens (PLAN.md:7-13). Start from AuthCache's write path; TokenStore would own rotation state and delegate storage.\nDepends on / blocked by: a product or security requirement for per-token state.\nRecommendation: 8A because a written trigger stops the next engineer from either re-adding it reflexively or missing the moment it is actually needed.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: one P4 line in TODOS.md versus relitigating the cut from scratch later.": "8A) Add to TODOS.md (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:22:21.211Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "c539345c-8ee2-47a7-a11a-248460aa0e92",
|
||||
"toolUseId": "toolu_01Pme9F9sGemn8TCxLQgU4Lw",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D9 \u2014 Next steps: Eng Review is CLEAR. This is a backend auth refactor with no UI scope, so /plan-design-review does not apply. No CEO review exists, but the plan changes no product direction. What next?\nRecommendation: 9A because the plan is implementation-ready and a CEO review would add little to a pure infrastructure refactor.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: start building now versus one more optional review pass on scope.",
|
||||
"header": "Next steps",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "9A) Ready to implement \u2014 run /ship when done (recommended)",
|
||||
"description": "Exit plan mode with the reviewed plan; implement T1\u2013T10 (record T3 regression fixtures first). \u2705 All required reviews complete and logged. \u2705 Tasks JSONL and QA test plan are already written for /autoplan and /qa. \u274c No second-opinion pass since codex reviews are disabled."
|
||||
},
|
||||
{
|
||||
"label": "9B) Run /plan-ceo-review first",
|
||||
"description": "Optional scope and strategy pass before implementing. \u2705 Catches product-level questions the eng review does not ask. \u2705 Adds a CEO row to the dashboard. \u274c Little product surface here; likely confirms the current scope."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D9 \u2014 Next steps: Eng Review is CLEAR. This is a backend auth refactor with no UI scope, so /plan-design-review does not apply. No CEO review exists, but the plan changes no product direction. What next?\nRecommendation: 9A because the plan is implementation-ready and a CEO review would add little to a pure infrastructure refactor.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nNet: start building now versus one more optional review pass on scope.": "9A) Ready to implement \u2014 run /ship when done (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:26:25.864Z"
|
||||
}
|
||||
],
|
||||
"actual": {
|
||||
"outcome": "ceiling_reached",
|
||||
"step0Count": 4,
|
||||
"reviewCount": 8,
|
||||
"administrativeCount": 0,
|
||||
"capture": {
|
||||
"skill": "plan-eng-review",
|
||||
"runId": "ship-source-ad-full-paid-20260909-v2-8",
|
||||
"cwd": "/tmp/gstack-paid-shard-T0nWrR/tmp/gstack-plan-count-5RsHax",
|
||||
"claudeConfigDir": "/tmp/gstack-paid-shard-T0nWrR/tmp/gstack-hermetic-1680405-MuryLZ/with-skills/.claude",
|
||||
"at": "2026-09-09T19:26:29.868Z"
|
||||
}
|
||||
},
|
||||
"sourceObservation": {
|
||||
"path": ".context/ship-source-ad-full-paid-20260909-v2/evals/job-8/shards/skill-e2e-plan-eng-finding-count/pty-count/ship-source-ad-full-paid-20260909-v2-8/plan-eng-review-1788981392626-sTIFTU/observation.json",
|
||||
"sha256": "73ff0c5fa0e872017ee52c050f8a3d733e3c884b89498ef158d4d5358da358e4"
|
||||
},
|
||||
"nativePrefix": {
|
||||
"path": "/home/vercel-sandbox/gstack/.context/ship-source-ad-full-paid-20260909-v2/full-pty-evidence/blobs/09692ed39e22896fec77e6e2473fc5902f0cd868b282ef4c002ed7272f8b10da/current.jsonl",
|
||||
"bytes": 745800,
|
||||
"sha256": "aeb735f46762ded51832d0f39746f409fef9a78ed1c00d43bd586024345da0d1"
|
||||
},
|
||||
"timeAnchors": {
|
||||
"toolu_011mVSdHDp1RpQqkGZAgUuQs": {
|
||||
"requestAt": "2026-09-09T19:16:49.391Z",
|
||||
"replyAt": "2026-09-09T19:16:50.663Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01JAQgq1i4dr7qQ1g8KY52Mn": {
|
||||
"requestAt": "2026-09-09T19:17:36.733Z",
|
||||
"replyAt": "2026-09-09T19:17:38.771Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01K24EDkSXH6E5Rhpoz5TF9o": {
|
||||
"requestAt": "2026-09-09T19:18:25.125Z",
|
||||
"replyAt": "2026-09-09T19:18:26.898Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01KJKgeVCDfHPaCE7PoyjdzA": {
|
||||
"requestAt": "2026-09-09T19:18:45.650Z",
|
||||
"replyAt": "2026-09-09T19:18:46.953Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01Ht48i1hSkowFxvDDK6Q1u1": {
|
||||
"requestAt": "2026-09-09T19:19:32.099Z",
|
||||
"replyAt": "2026-09-09T19:19:33.096Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01CMVw67fBzvHDC5T96DZF44": {
|
||||
"requestAt": "2026-09-09T19:19:53.130Z",
|
||||
"replyAt": "2026-09-09T19:19:55.162Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01WjXvB3hLuPx6wY48yDkmcr": {
|
||||
"requestAt": "2026-09-09T19:20:18.202Z",
|
||||
"replyAt": "2026-09-09T19:20:19.245Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01KkGg1niJ1a26DfcKzbdKCp": {
|
||||
"requestAt": "2026-09-09T19:20:43.126Z",
|
||||
"replyAt": "2026-09-09T19:20:43.329Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01HNpCxuxYumsrgJEm4LWYze": {
|
||||
"requestAt": "2026-09-09T19:21:14.421Z",
|
||||
"replyAt": "2026-09-09T19:21:15.450Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01DAdA2yiWKJF9KdbYZG4spf": {
|
||||
"requestAt": "2026-09-09T19:21:58.598Z",
|
||||
"replyAt": "2026-09-09T19:21:59.117Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01Vsi1PpqXVd72HWqZ3HD7do": {
|
||||
"requestAt": "2026-09-09T19:22:20.618Z",
|
||||
"replyAt": "2026-09-09T19:22:21.211Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01PBxaZDLPnveMvUUmMFJ8Jm": {
|
||||
"requestAt": "2026-09-09T19:25:26.587Z",
|
||||
"replyAt": "2026-09-09T19:25:28.211Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01Pme9F9sGemn8TCxLQgU4Lw": {
|
||||
"requestAt": "2026-09-09T19:26:24.896Z",
|
||||
"replyAt": "2026-09-09T19:26:25.864Z",
|
||||
"isError": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"retry": {
|
||||
"calls": [
|
||||
{
|
||||
"sessionId": "15480a19-46af-4e49-94b0-073b3ff142a3",
|
||||
"toolUseId": "toolu_015NKifTUKmcwcMTK1KkRWms",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D1 \u2014 Add gstack skill routing rules to this project's CLAUDE.md?\nProject/branch/task: main branch of the auth-refactor plan fixture, one-time gstack onboarding prompt.\nELI10: gstack skills work best when the project's CLAUDE.md tells the assistant which skill to reach for (bugs \u2192 /investigate, architecture \u2192 /plan-eng-review, and so on). Without it you invoke skills by name every time. This only asks once per project.\nStakes if we pick wrong: Low either way. Declining means manual skill invocation; accepting adds a short section to CLAUDE.md and a commit (deferred until plan mode ends, since edits are frozen right now).\nRecommendation: A because routing rules are cheap and make the skills fire when they should.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nPros / cons:\nA) Add routing rules to CLAUDE.md (recommended)\n \u2705 Skills auto-route from natural requests like 'review this architecture' without naming them\n \u2705 Teammates who clone the repo get the same routing behavior from the committed file\n \u274c Adds a section to CLAUDE.md and a commit; in plan mode this write is deferred until the review completes\nB) No thanks, I'll invoke skills manually\n \u2705 CLAUDE.md stays untouched and no extra commit lands on main\n \u2705 Full control over when each skill runs; nothing fires proactively\n \u274c You must remember and type each skill name; the prompt never reappears for this project\nNet: A trades one small committed file section for skills that route themselves.",
|
||||
"header": "Routing",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Add routing rules (recommended)",
|
||||
"description": "Append the gstack skill routing section to CLAUDE.md and commit it once plan mode allows edits."
|
||||
},
|
||||
{
|
||||
"label": "No thanks, manual",
|
||||
"description": "Mark routing as declined for this project; invoke skills by name."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D1 \u2014 Add gstack skill routing rules to this project's CLAUDE.md?\nProject/branch/task: main branch of the auth-refactor plan fixture, one-time gstack onboarding prompt.\nELI10: gstack skills work best when the project's CLAUDE.md tells the assistant which skill to reach for (bugs \u2192 /investigate, architecture \u2192 /plan-eng-review, and so on). Without it you invoke skills by name every time. This only asks once per project.\nStakes if we pick wrong: Low either way. Declining means manual skill invocation; accepting adds a short section to CLAUDE.md and a commit (deferred until plan mode ends, since edits are frozen right now).\nRecommendation: A because routing rules are cheap and make the skills fire when they should.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nPros / cons:\nA) Add routing rules to CLAUDE.md (recommended)\n \u2705 Skills auto-route from natural requests like 'review this architecture' without naming them\n \u2705 Teammates who clone the repo get the same routing behavior from the committed file\n \u274c Adds a section to CLAUDE.md and a commit; in plan mode this write is deferred until the review completes\nB) No thanks, I'll invoke skills manually\n \u2705 CLAUDE.md stays untouched and no extra commit lands on main\n \u2705 Full control over when each skill runs; nothing fires proactively\n \u274c You must remember and type each skill name; the prompt never reappears for this project\nNet: A trades one small committed file section for skills that route themselves.": "Add routing rules (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:27:14.423Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "15480a19-46af-4e49-94b0-073b3ff142a3",
|
||||
"toolUseId": "toolu_01WDfYnvm9gRgvyJh52pnbBw",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D2 \u2014 No design doc found. Run /office-hours first, or proceed with the standard eng review?\nProject/branch/task: main branch, reviewing PLAN.md (Multi-tenant Auth Refactor).\nELI10: A design doc captures the problem statement, the premise challenge, and the alternatives you already rejected. This review works from PLAN.md alone, which lists components and known smells but not the why. Office hours would produce that doc in about 10 minutes and give the review sharper input. The doc is per-feature, not per-product.\nStakes if we pick wrong: Skipping means some recommendations may re-litigate choices you already made for reasons the plan doesn't state. Running it costs ~10 minutes before the review starts.\nRecommendation: B because the plan already names its architecture, smells, and scope, and this is a refactor with a clear goal, so the review can proceed on what's written.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nPros / cons:\nA) Run /office-hours now (human: ~1h / CC: ~10 min)\n \u2705 Produces a problem statement and rejected alternatives the review can ground against\n \u2705 Surfaces whether the 4-class design is essential or accidental before we critique it\n \u274c Adds ~10 minutes before any review finding lands; the plan is already fairly specific\nB) Skip \u2014 proceed with standard review (recommended)\n \u2705 Review starts now against PLAN.md, which already names components, smells, and scope\n \u2705 Scope Challenge in Step 0 covers the 'is this overbuilt' question directly\n \u274c Recommendations may lack context on why four classes were chosen over fewer\nNet: A buys grounding at the cost of time; B trusts the plan as written and lets Step 0 do the scope challenge.",
|
||||
"header": "Design doc",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Skip, standard review (recommended)",
|
||||
"description": "Proceed directly to Step 0 Scope Challenge against PLAN.md."
|
||||
},
|
||||
{
|
||||
"label": "Run /office-hours now",
|
||||
"description": "Produce a design doc first, then resume this review."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D2 \u2014 No design doc found. Run /office-hours first, or proceed with the standard eng review?\nProject/branch/task: main branch, reviewing PLAN.md (Multi-tenant Auth Refactor).\nELI10: A design doc captures the problem statement, the premise challenge, and the alternatives you already rejected. This review works from PLAN.md alone, which lists components and known smells but not the why. Office hours would produce that doc in about 10 minutes and give the review sharper input. The doc is per-feature, not per-product.\nStakes if we pick wrong: Skipping means some recommendations may re-litigate choices you already made for reasons the plan doesn't state. Running it costs ~10 minutes before the review starts.\nRecommendation: B because the plan already names its architecture, smells, and scope, and this is a refactor with a clear goal, so the review can proceed on what's written.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nPros / cons:\nA) Run /office-hours now (human: ~1h / CC: ~10 min)\n \u2705 Produces a problem statement and rejected alternatives the review can ground against\n \u2705 Surfaces whether the 4-class design is essential or accidental before we critique it\n \u274c Adds ~10 minutes before any review finding lands; the plan is already fairly specific\nB) Skip \u2014 proceed with standard review (recommended)\n \u2705 Review starts now against PLAN.md, which already names components, smells, and scope\n \u2705 Scope Challenge in Step 0 covers the 'is this overbuilt' question directly\n \u274c Recommendations may lack context on why four classes were chosen over fewer\nNet: A buys grounding at the cost of time; B trusts the plan as written and lets Step 0 do the scope challenge.": "Skip, standard review (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:28:22.751Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "15480a19-46af-4e49-94b0-073b3ff142a3",
|
||||
"toolUseId": "toolu_01Ky3yywB4CuYtXaSoF9v6Wa",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D3 \u2014 Scope: 4 new classes across 12 files. Reduce to the core, or proceed as planned?\nProject/branch/task: main branch, PLAN.md Multi-tenant Auth Refactor, Step 0 complexity check (PLAN.md:35-36 names TokenStore, SessionMint, AuthCache, RequestPolicy).\nELI10: The plan adds four new classes to do multi-tenant auth. Two of them look like they duplicate things you already have. The existing cache adapter already stores tokens keyed by tenant, issuer, audience, and policy version (PLAN.md:7-13), so a separate TokenStore is a second place tokens live. RequestPolicy wraps a policy version that is already part of the cache key. Every extra class is another thing to test, another seam for bugs, and another file a 3am on-call engineer has to read.\nStakes if we pick wrong: Too big and you ship two token stores that can disagree about whether a token is valid, which is a tenant-isolation bug. Too small and you cram policy logic into the broker and refactor again in a month.\nRecommendation: A because AuthBroker, SessionMint, and the AuthCache facade are the essential complexity; TokenStore and RequestPolicy are accidental and fold into existing pieces with no loss of capability.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nPros / cons:\nA) Reduce: fold TokenStore into the AuthCache facade, make RequestPolicy a plain typed function or config object, keep AuthBroker + SessionMint (recommended) (human: ~1 day saved / CC: ~20 min saved)\n \u2705 One backing store for tokens means one place validity and tenant-key rules can be wrong, not two\n \u2705 Cuts the change to roughly 9 files and 3 classes, closer to the 'smallest diff that cleanly expresses the change' preference\n \u274c If policy later needs its own lifecycle (hot reload, per-tenant overrides), you promote the function to a class then\nB) Proceed as-is: 4 classes, 12 files\n \u2705 Every concept gets a named home, which reads clearly in code review and onboarding\n \u2705 No rework if TokenStore and RequestPolicy have responsibilities the plan text doesn't spell out\n \u274c Two token-holding layers (TokenStore and the existing adapter) can drift; more surface to test and to keep DRY\nC) Reduce harder: only AuthBroker + SessionMint, no AuthCache facade; both use the existing adapter directly (human: ~2 days saved / CC: ~30 min saved)\n \u2705 Smallest possible diff and zero new caching abstractions to learn\n \u2705 Reuses the adapter and its existing tests with no wrapper layer to mock\n \u274c Both services call adapter internals directly, so the tenant-key discipline lives in two call sites instead of one facade\nNet: A keeps the pieces that carry real behavior and deletes the two that echo existing code; B trusts the plan's decomposition; C trades a small facade for duplicated adapter calls.",
|
||||
"header": "Scope",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Reduce to 3 classes (recommended)",
|
||||
"description": "Fold TokenStore into AuthCache; RequestPolicy becomes a typed function/config. Keep AuthBroker, SessionMint, AuthCache."
|
||||
},
|
||||
{
|
||||
"label": "B) Proceed as-is",
|
||||
"description": "Keep all 4 classes and 12 files; review the plan as written."
|
||||
},
|
||||
{
|
||||
"label": "C) Reduce harder to 2 classes",
|
||||
"description": "Drop the AuthCache facade too; AuthBroker and SessionMint use the existing adapter directly."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D3 \u2014 Scope: 4 new classes across 12 files. Reduce to the core, or proceed as planned?\nProject/branch/task: main branch, PLAN.md Multi-tenant Auth Refactor, Step 0 complexity check (PLAN.md:35-36 names TokenStore, SessionMint, AuthCache, RequestPolicy).\nELI10: The plan adds four new classes to do multi-tenant auth. Two of them look like they duplicate things you already have. The existing cache adapter already stores tokens keyed by tenant, issuer, audience, and policy version (PLAN.md:7-13), so a separate TokenStore is a second place tokens live. RequestPolicy wraps a policy version that is already part of the cache key. Every extra class is another thing to test, another seam for bugs, and another file a 3am on-call engineer has to read.\nStakes if we pick wrong: Too big and you ship two token stores that can disagree about whether a token is valid, which is a tenant-isolation bug. Too small and you cram policy logic into the broker and refactor again in a month.\nRecommendation: A because AuthBroker, SessionMint, and the AuthCache facade are the essential complexity; TokenStore and RequestPolicy are accidental and fold into existing pieces with no loss of capability.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nPros / cons:\nA) Reduce: fold TokenStore into the AuthCache facade, make RequestPolicy a plain typed function or config object, keep AuthBroker + SessionMint (recommended) (human: ~1 day saved / CC: ~20 min saved)\n \u2705 One backing store for tokens means one place validity and tenant-key rules can be wrong, not two\n \u2705 Cuts the change to roughly 9 files and 3 classes, closer to the 'smallest diff that cleanly expresses the change' preference\n \u274c If policy later needs its own lifecycle (hot reload, per-tenant overrides), you promote the function to a class then\nB) Proceed as-is: 4 classes, 12 files\n \u2705 Every concept gets a named home, which reads clearly in code review and onboarding\n \u2705 No rework if TokenStore and RequestPolicy have responsibilities the plan text doesn't spell out\n \u274c Two token-holding layers (TokenStore and the existing adapter) can drift; more surface to test and to keep DRY\nC) Reduce harder: only AuthBroker + SessionMint, no AuthCache facade; both use the existing adapter directly (human: ~2 days saved / CC: ~30 min saved)\n \u2705 Smallest possible diff and zero new caching abstractions to learn\n \u2705 Reuses the adapter and its existing tests with no wrapper layer to mock\n \u274c Both services call adapter internals directly, so the tenant-key discipline lives in two call sites instead of one facade\nNet: A keeps the pieces that carry real behavior and deletes the two that echo existing code; B trusts the plan's decomposition; C trades a small facade for duplicated adapter calls.": "A) Reduce to 3 classes (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:29:31.125Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "15480a19-46af-4e49-94b0-073b3ff142a3",
|
||||
"toolUseId": "toolu_01NX2VMYiivofKtZWKv221zU",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Issue 1 (D4) \u2014 AuthCache is a global mutable module-level export that two services mutate. Inject it instead?\nProject/branch/task: main branch, PLAN.md:19-20: 'Two new services (AuthBroker and SessionMint) share a global mutable AuthCache instance via module-level export. Both services mutate it.'\nELI10: Right now the cache is a global variable that any file can import and change. That means tests can't give each service a fresh cache (Jest's module cache makes the same instance leak between test files), nothing stops a third module from importing and poking it, and it hides that AuthBroker and SessionMint depend on shared state. The fix is boring: build one AuthCache in the app's startup code and hand it to each service's constructor. [Layer 1: constructor injection from a composition root is the documented standard for this.]\nStakes if we pick wrong: Flaky auth tests that pass alone and fail together, and a tenant-isolation bug that is hard to reproduce because the shared instance carries state across requests.\nRecommendation: 1A because explicit dependencies beat clever globals, and it makes every AuthCache test able to start from a clean instance.\nCompleteness: A=10/10, B=6/10, C=2/10\nPros / cons:\n1A) Constructor-inject one AuthCache from the composition root; services only call its typed methods, no direct export (recommended) (human: ~3h / CC: ~15 min)\n \u2705 Each test constructs its own AuthCache, so cross-test state leakage disappears entirely\n \u2705 The dependency graph is visible in constructor signatures, which matches the explicit-over-clever preference\n \u274c Wiring code in the composition root grows by a few lines and existing callers of the export need updating\n1B) Keep the module export but expose it behind a getter with a test-only reset hook (human: ~1h / CC: ~5 min)\n \u2705 Minimal churn to the files that already import the module-level instance\n \u2705 Tests can reset state between cases via the hook\n \u274c Hidden global coupling stays, and a reset hook shipping in production code is a footgun\n1C) Do nothing; keep the shared module-level instance\n \u2705 Zero implementation cost right now\n \u2705 Works fine while only two well-behaved services touch it\n \u274c Any new importer can mutate auth state; test isolation depends on module cache behavior you don't control\nNet: 1A is a small wiring change that buys deterministic tests and a visible dependency graph; 1B papers over it; 1C defers the bug to whoever adds the third importer.",
|
||||
"header": "Arch 1",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "1A) Constructor-inject (recommended)",
|
||||
"description": "One AuthCache built at startup, passed into AuthBroker and SessionMint; remove the module-level export."
|
||||
},
|
||||
{
|
||||
"label": "1B) Getter + reset hook",
|
||||
"description": "Keep the export but wrap access and add a test reset."
|
||||
},
|
||||
{
|
||||
"label": "1C) Do nothing",
|
||||
"description": "Keep the shared module-level mutable instance as planned."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"Issue 1 (D4) \u2014 AuthCache is a global mutable module-level export that two services mutate. Inject it instead?\nProject/branch/task: main branch, PLAN.md:19-20: 'Two new services (AuthBroker and SessionMint) share a global mutable AuthCache instance via module-level export. Both services mutate it.'\nELI10: Right now the cache is a global variable that any file can import and change. That means tests can't give each service a fresh cache (Jest's module cache makes the same instance leak between test files), nothing stops a third module from importing and poking it, and it hides that AuthBroker and SessionMint depend on shared state. The fix is boring: build one AuthCache in the app's startup code and hand it to each service's constructor. [Layer 1: constructor injection from a composition root is the documented standard for this.]\nStakes if we pick wrong: Flaky auth tests that pass alone and fail together, and a tenant-isolation bug that is hard to reproduce because the shared instance carries state across requests.\nRecommendation: 1A because explicit dependencies beat clever globals, and it makes every AuthCache test able to start from a clean instance.\nCompleteness: A=10/10, B=6/10, C=2/10\nPros / cons:\n1A) Constructor-inject one AuthCache from the composition root; services only call its typed methods, no direct export (recommended) (human: ~3h / CC: ~15 min)\n \u2705 Each test constructs its own AuthCache, so cross-test state leakage disappears entirely\n \u2705 The dependency graph is visible in constructor signatures, which matches the explicit-over-clever preference\n \u274c Wiring code in the composition root grows by a few lines and existing callers of the export need updating\n1B) Keep the module export but expose it behind a getter with a test-only reset hook (human: ~1h / CC: ~5 min)\n \u2705 Minimal churn to the files that already import the module-level instance\n \u2705 Tests can reset state between cases via the hook\n \u274c Hidden global coupling stays, and a reset hook shipping in production code is a footgun\n1C) Do nothing; keep the shared module-level instance\n \u2705 Zero implementation cost right now\n \u2705 Works fine while only two well-behaved services touch it\n \u274c Any new importer can mutate auth state; test isolation depends on module cache behavior you don't control\nNet: 1A is a small wiring change that buys deterministic tests and a visible dependency graph; 1B papers over it; 1C defers the bug to whoever adds the third importer.": "1A) Constructor-inject (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:30:05.310Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "15480a19-46af-4e49-94b0-073b3ff142a3",
|
||||
"toolUseId": "toolu_01L9PAk5ZfhJ53W9YgDDJekn",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Issue 2 (D5) \u2014 Two writers, no serialization: a SessionMint write can land after an AuthBroker invalidation and resurrect a revoked token. Guard it?\nProject/branch/task: main branch, PLAN.md:10 'they do not serialize mutations' and PLAN.md:20 'Both services mutate it'; invalidation triggers at PLAN.md:8-9 (logout, revocation, tenant suspension).\nELI10: Picture this sequence. SessionMint starts minting a session for tenant T and reads the policy. Meanwhile an admin suspends tenant T, and AuthBroker invalidates every cache entry for T. Then SessionMint finishes and writes its fresh session entry. The cache now holds a valid-looking token for a suspended tenant. Nothing in the plan orders these two writes. The fix is a per-tenant invalidation epoch: AuthCache records an epoch counter bumped on every invalidate, writers capture the epoch when they start, and AuthCache.put rejects a write whose epoch is stale. This lives inside the facade the user just approved, so services don't change.\nStakes if we pick wrong: A suspended or logged-out tenant keeps a live token until natural expiry. That is a security incident, not a bug ticket.\nRecommendation: 2A because the guard is one integer compare inside the facade, and the failure it prevents is silent and security-relevant.\nCompleteness: A=10/10, B=7/10, C=1/10\nPros / cons:\n2A) Per-tenant invalidation epoch in AuthCache; put() rejects stale-epoch writes and returns a typed StaleWrite result; test the interleaving explicitly (recommended) (human: ~4h / CC: ~20 min)\n \u2705 Closes the write-after-invalidate window for every current and future writer, not just SessionMint\n \u2705 Rejected writes are explicit and testable, so the race has a deterministic unit test instead of a flaky one\n \u274c Adds an epoch map to the facade and one more return branch callers must handle\n2B) Re-check tenant status inside SessionMint immediately before its write (human: ~1h / CC: ~5 min)\n \u2705 Small, local change with no new state in the cache\n \u2705 Catches the common suspension case in practice\n \u274c Still a check-then-act race with a smaller window; every future writer must remember to copy the check\n2C) Do nothing; rely on token TTL to expire the stale entry\n \u2705 No code; the entry eventually dies on its own\n \u2705 Acceptable if TTLs are seconds and suspension is rare\n \u274c A revoked token stays usable for the full TTL, and logout-then-reuse becomes reproducible by an attacker\nNet: 2A fixes the race once in the shared facade; 2B shrinks the window per writer; 2C accepts a revocation gap.",
|
||||
"header": "Arch 2",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "2A) Invalidation epoch in AuthCache (recommended)",
|
||||
"description": "Epoch per tenant; stale-epoch puts rejected with a typed result; interleaving test included."
|
||||
},
|
||||
{
|
||||
"label": "2B) Re-check before write",
|
||||
"description": "SessionMint re-reads tenant status right before its cache write."
|
||||
},
|
||||
{
|
||||
"label": "2C) Do nothing",
|
||||
"description": "Accept the revocation gap until TTL expiry."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"Issue 2 (D5) \u2014 Two writers, no serialization: a SessionMint write can land after an AuthBroker invalidation and resurrect a revoked token. Guard it?\nProject/branch/task: main branch, PLAN.md:10 'they do not serialize mutations' and PLAN.md:20 'Both services mutate it'; invalidation triggers at PLAN.md:8-9 (logout, revocation, tenant suspension).\nELI10: Picture this sequence. SessionMint starts minting a session for tenant T and reads the policy. Meanwhile an admin suspends tenant T, and AuthBroker invalidates every cache entry for T. Then SessionMint finishes and writes its fresh session entry. The cache now holds a valid-looking token for a suspended tenant. Nothing in the plan orders these two writes. The fix is a per-tenant invalidation epoch: AuthCache records an epoch counter bumped on every invalidate, writers capture the epoch when they start, and AuthCache.put rejects a write whose epoch is stale. This lives inside the facade the user just approved, so services don't change.\nStakes if we pick wrong: A suspended or logged-out tenant keeps a live token until natural expiry. That is a security incident, not a bug ticket.\nRecommendation: 2A because the guard is one integer compare inside the facade, and the failure it prevents is silent and security-relevant.\nCompleteness: A=10/10, B=7/10, C=1/10\nPros / cons:\n2A) Per-tenant invalidation epoch in AuthCache; put() rejects stale-epoch writes and returns a typed StaleWrite result; test the interleaving explicitly (recommended) (human: ~4h / CC: ~20 min)\n \u2705 Closes the write-after-invalidate window for every current and future writer, not just SessionMint\n \u2705 Rejected writes are explicit and testable, so the race has a deterministic unit test instead of a flaky one\n \u274c Adds an epoch map to the facade and one more return branch callers must handle\n2B) Re-check tenant status inside SessionMint immediately before its write (human: ~1h / CC: ~5 min)\n \u2705 Small, local change with no new state in the cache\n \u2705 Catches the common suspension case in practice\n \u274c Still a check-then-act race with a smaller window; every future writer must remember to copy the check\n2C) Do nothing; rely on token TTL to expire the stale entry\n \u2705 No code; the entry eventually dies on its own\n \u2705 Acceptable if TTLs are seconds and suspension is rare\n \u274c A revoked token stays usable for the full TTL, and logout-then-reuse becomes reproducible by an attacker\nNet: 2A fixes the race once in the shared facade; 2B shrinks the window per writer; 2C accepts a revocation gap.": "2A) Invalidation epoch in AuthCache (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:30:31.467Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "15480a19-46af-4e49-94b0-073b3ff142a3",
|
||||
"toolUseId": "toolu_01APA6Bw9GjU4zMRRzix7bYt",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Issue 3 (D6) \u2014 validateAndDispatch() is 60 lines with three nested try/catch blocks that each swallow a different error class. Restructure it?\nProject/branch/task: main branch, PLAN.md:23-24: 'The validateAndDispatch() function is 60 lines with three nested try/catch blocks; each catch swallows a different error class.'\nELI10: Three nested try/catch blocks that swallow errors means three places where a real failure turns into silence. If the token signature check throws and that catch eats it, the function keeps going and may dispatch a request that should have been rejected. The fix is to split the function into a validate step and a dispatch step, make each error type a named result instead of a swallowed exception, and let one top-level handler decide what the caller sees. A single flat pipeline is also far easier to test branch by branch.\nStakes if we pick wrong: Auth failures get logged as nothing and dispatched as success. The user sees a request that should have been a 401 succeed, or a confusing generic error with no cause.\nRecommendation: 3A because swallowing errors in an auth path violates explicit-over-clever and blocks 100% branch coverage; the split is a pure refactor with no behavior change in the happy path.\nCompleteness: A=10/10, B=6/10, C=1/10\nPros / cons:\n3A) Split into validate() returning a typed Result (Ok | SignatureError | ClaimsError | PolicyError) and dispatch(); one top-level handler maps each error to a logged, user-visible outcome; unit test every variant (recommended) (human: ~4h / CC: ~20 min)\n \u2705 Every error class becomes a visible branch with its own test instead of a silent catch\n \u2705 Each half stays under 20 lines, and the same validate() is reusable by AuthBroker and SessionMint (DRY)\n \u274c Callers change from one call to two, and the typed Result adds a small type definition\n3B) Keep one function but flatten the try/catch blocks and log every swallowed error (human: ~1h / CC: ~5 min)\n \u2705 Small diff; errors at least show up in logs\n \u2705 No caller changes\n \u274c Still a 60-line function mixing validation and dispatch, and logging a swallowed error does not stop the wrong dispatch\n3C) Do nothing; leave the function as-is\n \u2705 Zero cost now and no risk of a refactor typo\n \u2705 Presumably works for the happy path today\n \u274c Silent error swallowing in the auth path stays, and the nested structure cannot be fully branch-tested\nNet: 3A turns hidden failures into typed, tested branches; 3B adds visibility without fixing control flow; 3C keeps a known landmine.",
|
||||
"header": "Code Q 3",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "3A) Split + typed Result (recommended)",
|
||||
"description": "validate() returns a discriminated Result; dispatch() is separate; one handler maps errors; every variant tested."
|
||||
},
|
||||
{
|
||||
"label": "3B) Flatten + log",
|
||||
"description": "Keep the function, flatten the blocks, log each swallowed error."
|
||||
},
|
||||
{
|
||||
"label": "3C) Do nothing",
|
||||
"description": "Leave validateAndDispatch() as described."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"Issue 3 (D6) \u2014 validateAndDispatch() is 60 lines with three nested try/catch blocks that each swallow a different error class. Restructure it?\nProject/branch/task: main branch, PLAN.md:23-24: 'The validateAndDispatch() function is 60 lines with three nested try/catch blocks; each catch swallows a different error class.'\nELI10: Three nested try/catch blocks that swallow errors means three places where a real failure turns into silence. If the token signature check throws and that catch eats it, the function keeps going and may dispatch a request that should have been rejected. The fix is to split the function into a validate step and a dispatch step, make each error type a named result instead of a swallowed exception, and let one top-level handler decide what the caller sees. A single flat pipeline is also far easier to test branch by branch.\nStakes if we pick wrong: Auth failures get logged as nothing and dispatched as success. The user sees a request that should have been a 401 succeed, or a confusing generic error with no cause.\nRecommendation: 3A because swallowing errors in an auth path violates explicit-over-clever and blocks 100% branch coverage; the split is a pure refactor with no behavior change in the happy path.\nCompleteness: A=10/10, B=6/10, C=1/10\nPros / cons:\n3A) Split into validate() returning a typed Result (Ok | SignatureError | ClaimsError | PolicyError) and dispatch(); one top-level handler maps each error to a logged, user-visible outcome; unit test every variant (recommended) (human: ~4h / CC: ~20 min)\n \u2705 Every error class becomes a visible branch with its own test instead of a silent catch\n \u2705 Each half stays under 20 lines, and the same validate() is reusable by AuthBroker and SessionMint (DRY)\n \u274c Callers change from one call to two, and the typed Result adds a small type definition\n3B) Keep one function but flatten the try/catch blocks and log every swallowed error (human: ~1h / CC: ~5 min)\n \u2705 Small diff; errors at least show up in logs\n \u2705 No caller changes\n \u274c Still a 60-line function mixing validation and dispatch, and logging a swallowed error does not stop the wrong dispatch\n3C) Do nothing; leave the function as-is\n \u2705 Zero cost now and no risk of a refactor typo\n \u2705 Presumably works for the happy path today\n \u274c Silent error swallowing in the auth path stays, and the nested structure cannot be fully branch-tested\nNet: 3A turns hidden failures into typed, tested branches; 3B adds visibility without fixing control flow; 3C keeps a known landmine.": "3A) Split + typed Result (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:30:57.623Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "15480a19-46af-4e49-94b0-073b3ff142a3",
|
||||
"toolUseId": "toolu_01XnrCrsR4sNA1aVWCroDoKA",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Issue 4 (D7) \u2014 Planned coverage is unit + integration on the new components only. Add an end-to-end auth journey test across two tenants?\nProject/branch/task: main branch, PLAN.md:14-15: 'Unit and integration coverage is planned for the new components and their success/error paths.' No test walks the full flow through AuthBroker, SessionMint, and AuthCache together.\nELI10: Unit tests prove each box works alone. This change wires three boxes plus an external identity provider (IDP, the service that issues tokens) into one flow. The bugs that hurt in multi-tenant auth live in the seams: tenant A's session showing up for tenant B, or a logged-out token still working. An end-to-end test spins up two tenants, logs in as each, mints sessions, validates, logs one out, and asserts the other is untouched and the logged-out token is rejected. Auth flows are on the skill's always-E2E list because mocking hides exactly these failures.\nStakes if we pick wrong: A cross-tenant leak or revocation gap ships with green unit tests, and you find out from a customer.\nRecommendation: 4A because the marginal cost of an E2E over the integration tests already planned is small with CC, and it is the only test that proves tenant isolation end to end.\nCompleteness: A=10/10, B=7/10, C=4/10\nPros / cons:\n4A) Add an E2E journey test: two tenants, login \u2192 mint \u2192 validate \u2192 logout \u2192 reuse rejected, plus tenant-suspension mid-flight; runs against a stubbed IDP in CI (recommended) (human: ~1 day / CC: ~30 min)\n \u2705 Proves tenant isolation and revocation through the real wiring, not through mocks that agree with themselves\n \u2705 Exercises the invalidation-epoch guard (2A) and the typed validate Result (3A) in one place\n \u274c Needs an IDP stub fixture and adds the slowest test in the suite\n4B) Integration tests only, but add one cross-tenant isolation case at the AuthCache facade level (human: ~2h / CC: ~10 min)\n \u2705 Cheap, fast, and catches key-construction mistakes in the facade\n \u2705 No IDP stub required\n \u274c Does not exercise AuthBroker and SessionMint together, so the write-after-invalidate race is never hit end to end\n4C) Keep the plan's coverage as written\n \u2705 No additional test infrastructure\n \u2705 Unit and integration coverage already covers success and error paths per component\n \u274c Seam bugs between the three components and the IDP stay untested\nNet: 4A buys the one test that proves the security property; 4B covers the facade; 4C trusts the seams.",
|
||||
"header": "Tests 4",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "4A) Add E2E journey test (recommended)",
|
||||
"description": "Two-tenant end-to-end flow with stubbed IDP, including logout reuse and mid-flight suspension."
|
||||
},
|
||||
{
|
||||
"label": "4B) Facade isolation test only",
|
||||
"description": "One cross-tenant isolation case at the AuthCache level; no E2E."
|
||||
},
|
||||
{
|
||||
"label": "4C) Keep as written",
|
||||
"description": "Unit and integration coverage on new components only."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"Issue 4 (D7) \u2014 Planned coverage is unit + integration on the new components only. Add an end-to-end auth journey test across two tenants?\nProject/branch/task: main branch, PLAN.md:14-15: 'Unit and integration coverage is planned for the new components and their success/error paths.' No test walks the full flow through AuthBroker, SessionMint, and AuthCache together.\nELI10: Unit tests prove each box works alone. This change wires three boxes plus an external identity provider (IDP, the service that issues tokens) into one flow. The bugs that hurt in multi-tenant auth live in the seams: tenant A's session showing up for tenant B, or a logged-out token still working. An end-to-end test spins up two tenants, logs in as each, mints sessions, validates, logs one out, and asserts the other is untouched and the logged-out token is rejected. Auth flows are on the skill's always-E2E list because mocking hides exactly these failures.\nStakes if we pick wrong: A cross-tenant leak or revocation gap ships with green unit tests, and you find out from a customer.\nRecommendation: 4A because the marginal cost of an E2E over the integration tests already planned is small with CC, and it is the only test that proves tenant isolation end to end.\nCompleteness: A=10/10, B=7/10, C=4/10\nPros / cons:\n4A) Add an E2E journey test: two tenants, login \u2192 mint \u2192 validate \u2192 logout \u2192 reuse rejected, plus tenant-suspension mid-flight; runs against a stubbed IDP in CI (recommended) (human: ~1 day / CC: ~30 min)\n \u2705 Proves tenant isolation and revocation through the real wiring, not through mocks that agree with themselves\n \u2705 Exercises the invalidation-epoch guard (2A) and the typed validate Result (3A) in one place\n \u274c Needs an IDP stub fixture and adds the slowest test in the suite\n4B) Integration tests only, but add one cross-tenant isolation case at the AuthCache facade level (human: ~2h / CC: ~10 min)\n \u2705 Cheap, fast, and catches key-construction mistakes in the facade\n \u2705 No IDP stub required\n \u274c Does not exercise AuthBroker and SessionMint together, so the write-after-invalidate race is never hit end to end\n4C) Keep the plan's coverage as written\n \u2705 No additional test infrastructure\n \u2705 Unit and integration coverage already covers success and error paths per component\n \u274c Seam bugs between the three components and the IDP stay untested\nNet: 4A buys the one test that proves the security property; 4B covers the facade; 4C trusts the seams.": "4A) Add E2E journey test (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:31:41.899Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "15480a19-46af-4e49-94b0-073b3ff142a3",
|
||||
"toolUseId": "toolu_01Gc8Bqbok4jzu98v6ocb3LG",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Issue 5 (D8) \u2014 Token validation makes 5 sequential IDP calls that are independent. Parallelize, and with what failure semantics?\nProject/branch/task: main branch, PLAN.md:31-32: 'Token validation issues 5 sequential API calls to the IDP; they could be parallelized via Promise.all trivially (calls are independent).'\nELI10: Five network calls one after another means the user waits five round trips on every login. Running them at once cuts that to one round trip. But Promise.all alone has a catch: the first failure rejects immediately and the other four calls keep running in the background with nobody listening, and a hung call hangs the whole login. The complete version wraps each call in a timeout, runs them together, and maps each call's failure into the typed validate Result from Issue 3 so the user sees which check failed. [Layer 1: Promise.all and AbortSignal.timeout are built-ins; no library needed.]\nStakes if we pick wrong: Either logins stay 5x slower than necessary, or a naive Promise.all turns one slow IDP endpoint into hung logins with no useful error.\nRecommendation: 5A because the plan already calls the parallelization trivial; the timeout and error mapping are the difference between fast and fast-but-fragile.\nCompleteness: A=10/10, B=6/10, C=2/10\nPros / cons:\n5A) Promise.all with per-call AbortSignal.timeout, each failure mapped to a typed validate Result variant, plus tests for one-call failure, one-call timeout, and all-success (recommended) (human: ~3h / CC: ~15 min)\n \u2705 Login latency drops from 5 round trips to 1 with a hard upper bound per call\n \u2705 Each of the 5 checks failing produces a distinct, tested rejection instead of a generic error\n \u274c Slightly more code than a bare Promise.all and one more test file\n5B) Bare Promise.all, no timeouts, first rejection wins (human: ~30 min / CC: ~3 min)\n \u2705 One-line change that gets the latency win immediately\n \u2705 Easy to read and matches what the plan already proposes\n \u274c A hung IDP call hangs the login, and the rejection loses which check failed\n5C) Leave the calls sequential\n \u2705 Zero change; failure ordering stays simple and deterministic\n \u2705 No concurrency to reason about\n \u274c Every login pays 5 serial round trips for no reason; the plan itself calls the fix trivial\nNet: 5A is the parallelization the plan wants with the two guardrails that make it safe; 5B gets the speed and a new hang risk; 5C keeps a known slowdown.",
|
||||
"header": "Perf 5",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "5A) Promise.all + timeouts + typed errors (recommended)",
|
||||
"description": "Per-call AbortSignal.timeout, failures mapped to validate Result variants, three tests."
|
||||
},
|
||||
{
|
||||
"label": "5B) Bare Promise.all",
|
||||
"description": "Parallelize with no timeouts; first rejection wins."
|
||||
},
|
||||
{
|
||||
"label": "5C) Keep sequential",
|
||||
"description": "Leave the 5 calls as they are."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"Issue 5 (D8) \u2014 Token validation makes 5 sequential IDP calls that are independent. Parallelize, and with what failure semantics?\nProject/branch/task: main branch, PLAN.md:31-32: 'Token validation issues 5 sequential API calls to the IDP; they could be parallelized via Promise.all trivially (calls are independent).'\nELI10: Five network calls one after another means the user waits five round trips on every login. Running them at once cuts that to one round trip. But Promise.all alone has a catch: the first failure rejects immediately and the other four calls keep running in the background with nobody listening, and a hung call hangs the whole login. The complete version wraps each call in a timeout, runs them together, and maps each call's failure into the typed validate Result from Issue 3 so the user sees which check failed. [Layer 1: Promise.all and AbortSignal.timeout are built-ins; no library needed.]\nStakes if we pick wrong: Either logins stay 5x slower than necessary, or a naive Promise.all turns one slow IDP endpoint into hung logins with no useful error.\nRecommendation: 5A because the plan already calls the parallelization trivial; the timeout and error mapping are the difference between fast and fast-but-fragile.\nCompleteness: A=10/10, B=6/10, C=2/10\nPros / cons:\n5A) Promise.all with per-call AbortSignal.timeout, each failure mapped to a typed validate Result variant, plus tests for one-call failure, one-call timeout, and all-success (recommended) (human: ~3h / CC: ~15 min)\n \u2705 Login latency drops from 5 round trips to 1 with a hard upper bound per call\n \u2705 Each of the 5 checks failing produces a distinct, tested rejection instead of a generic error\n \u274c Slightly more code than a bare Promise.all and one more test file\n5B) Bare Promise.all, no timeouts, first rejection wins (human: ~30 min / CC: ~3 min)\n \u2705 One-line change that gets the latency win immediately\n \u2705 Easy to read and matches what the plan already proposes\n \u274c A hung IDP call hangs the login, and the rejection loses which check failed\n5C) Leave the calls sequential\n \u2705 Zero change; failure ordering stays simple and deterministic\n \u2705 No concurrency to reason about\n \u274c Every login pays 5 serial round trips for no reason; the plan itself calls the fix trivial\nNet: 5A is the parallelization the plan wants with the two guardrails that make it safe; 5B gets the speed and a new hang risk; 5C keeps a known slowdown.": "5A) Promise.all + timeouts + typed errors (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:32:24.184Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "15480a19-46af-4e49-94b0-073b3ff142a3",
|
||||
"toolUseId": "toolu_01GS78UTYiYA9Y5cXNtwN6on",
|
||||
"questions": [
|
||||
{
|
||||
"question": "TODO 1 (D9) \u2014 Cache the IDP discovery document and JWKS so repeat validations skip the network. Capture it as a TODO, skip, or build now?\nProject/branch/task: main branch, PLAN.md:31 names 5 IDP calls per token validation; the plan does not say which calls they are.\nELI10: Most identity providers expose two things that barely change: a discovery document (where the endpoints are) and a JWKS (the public keys used to check token signatures). If any of the 5 calls fetch those, they can be cached for minutes to hours and refreshed on an unknown key id, with a rate limit so an attacker cannot force refetches. That turns 5 calls into 1 or 2 on the hot path. Medium confidence, verify this is actually an issue: I cannot see which 5 calls the plan means, so this may not apply.\nWhat: Add a bounded-TTL cache for IDP discovery + JWKS with unknown-kid refresh and a refetch rate limit.\nWhy: Removes network round trips from every validation and removes the IDP as a hot-path dependency for signature checks.\nPros: Faster validation and fewer IDP outages felt by users. Cons: Key-rotation edge cases (stale key, refresh storms) need their own tests. Context: Start in the validate() step from Issue 3; the cache belongs next to AuthCache but keyed by issuer, not tenant. Depends on: Issue 5's parallelized validate landing first so the call shape is settled.\nStakes if we pick wrong: Skipping leaves latency on the table if the calls are cacheable; building now widens this PR before the call shape is known.\nRecommendation: A because it is valuable but depends on knowing which 5 calls exist, which this PR will make clear.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nPros / cons:\nA) Add to TODOS.md (recommended)\n \u2705 Captures the idea with its rationale and starting point before the context is lost\n \u2705 Keeps this PR focused on the approved 3-class refactor\n \u274c Adds a backlog item that could sit unattended if nobody owns it\nB) Skip \u2014 not valuable enough\n \u2705 No backlog noise if the 5 calls turn out not to be cacheable\n \u2705 Zero effort now\n \u274c If they are cacheable, the latency win is forgotten until someone rediscovers it\nC) Build it now in this PR (human: ~1 day / CC: ~30 min)\n \u2705 Validation gets the full latency win in one change\n \u2705 Rotation tests land alongside the validate() refactor while the code is fresh\n \u274c Expands an already 9-file PR and depends on facts about the 5 calls the plan does not state\nNet: A records it for when the call shape is known; C bets the calls are cacheable; B drops it.",
|
||||
"header": "TODO 1",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Add to TODOS.md (recommended)",
|
||||
"description": "Record the discovery/JWKS caching idea with context, after this PR lands."
|
||||
},
|
||||
{
|
||||
"label": "B) Skip",
|
||||
"description": "Do not capture; revisit only if latency is still a problem."
|
||||
},
|
||||
{
|
||||
"label": "C) Build now in this PR",
|
||||
"description": "Add the cache in this change alongside the validate() refactor."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"TODO 1 (D9) \u2014 Cache the IDP discovery document and JWKS so repeat validations skip the network. Capture it as a TODO, skip, or build now?\nProject/branch/task: main branch, PLAN.md:31 names 5 IDP calls per token validation; the plan does not say which calls they are.\nELI10: Most identity providers expose two things that barely change: a discovery document (where the endpoints are) and a JWKS (the public keys used to check token signatures). If any of the 5 calls fetch those, they can be cached for minutes to hours and refreshed on an unknown key id, with a rate limit so an attacker cannot force refetches. That turns 5 calls into 1 or 2 on the hot path. Medium confidence, verify this is actually an issue: I cannot see which 5 calls the plan means, so this may not apply.\nWhat: Add a bounded-TTL cache for IDP discovery + JWKS with unknown-kid refresh and a refetch rate limit.\nWhy: Removes network round trips from every validation and removes the IDP as a hot-path dependency for signature checks.\nPros: Faster validation and fewer IDP outages felt by users. Cons: Key-rotation edge cases (stale key, refresh storms) need their own tests. Context: Start in the validate() step from Issue 3; the cache belongs next to AuthCache but keyed by issuer, not tenant. Depends on: Issue 5's parallelized validate landing first so the call shape is settled.\nStakes if we pick wrong: Skipping leaves latency on the table if the calls are cacheable; building now widens this PR before the call shape is known.\nRecommendation: A because it is valuable but depends on knowing which 5 calls exist, which this PR will make clear.\nNote: options differ in kind, not coverage \u2014 no completeness score.\nPros / cons:\nA) Add to TODOS.md (recommended)\n \u2705 Captures the idea with its rationale and starting point before the context is lost\n \u2705 Keeps this PR focused on the approved 3-class refactor\n \u274c Adds a backlog item that could sit unattended if nobody owns it\nB) Skip \u2014 not valuable enough\n \u2705 No backlog noise if the 5 calls turn out not to be cacheable\n \u2705 Zero effort now\n \u274c If they are cacheable, the latency win is forgotten until someone rediscovers it\nC) Build it now in this PR (human: ~1 day / CC: ~30 min)\n \u2705 Validation gets the full latency win in one change\n \u2705 Rotation tests land alongside the validate() refactor while the code is fresh\n \u274c Expands an already 9-file PR and depends on facts about the 5 calls the plan does not state\nNet: A records it for when the call shape is known; C bets the calls are cacheable; B drops it.": "A) Add to TODOS.md (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T19:33:02.445Z"
|
||||
}
|
||||
],
|
||||
"actual": {
|
||||
"outcome": "no_review_questions",
|
||||
"step0Count": 9,
|
||||
"reviewCount": 0,
|
||||
"administrativeCount": 0,
|
||||
"capture": {
|
||||
"skill": "plan-eng-review",
|
||||
"runId": "ship-source-ad-full-paid-20260909-v2-8",
|
||||
"cwd": "/tmp/gstack-paid-shard-T0nWrR/tmp/gstack-plan-count-XUA4hF",
|
||||
"claudeConfigDir": "/tmp/gstack-paid-shard-T0nWrR/tmp/gstack-hermetic-1680405-MuryLZ/with-skills/.claude",
|
||||
"at": "2026-09-09T19:37:29.864Z"
|
||||
}
|
||||
},
|
||||
"sourceObservation": {
|
||||
"path": ".context/ship-source-ad-full-paid-20260909-v2/evals/job-8/shards/skill-e2e-plan-eng-finding-count/pty-count/ship-source-ad-full-paid-20260909-v2-8/plan-eng-review-1788982020355-aesys4/observation.json",
|
||||
"sha256": "c1fd18a86bc194d216995d28ba94cc4eb45990f8cf9bed30b4248c1c146f5bd2"
|
||||
},
|
||||
"nativePrefix": {
|
||||
"path": "/home/vercel-sandbox/gstack/.context/ship-source-ad-full-paid-20260909-v2/full-pty-evidence/blobs/eb525d0cf32f0021b992137df56f1259f55d6812212f11e60d106fef464ff65d/current.jsonl",
|
||||
"bytes": 798793,
|
||||
"sha256": "c0ba6174952cf30dcbbcad36d57c41b5a804706d1e0558b3927491ae1e88b8d6"
|
||||
},
|
||||
"timeAnchors": {
|
||||
"toolu_015NKifTUKmcwcMTK1KkRWms": {
|
||||
"requestAt": "2026-09-09T19:27:13.096Z",
|
||||
"replyAt": "2026-09-09T19:27:14.423Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01WDfYnvm9gRgvyJh52pnbBw": {
|
||||
"requestAt": "2026-09-09T19:28:21.615Z",
|
||||
"replyAt": "2026-09-09T19:28:22.751Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01Ky3yywB4CuYtXaSoF9v6Wa": {
|
||||
"requestAt": "2026-09-09T19:29:29.446Z",
|
||||
"replyAt": "2026-09-09T19:29:31.125Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01NX2VMYiivofKtZWKv221zU": {
|
||||
"requestAt": "2026-09-09T19:30:05.102Z",
|
||||
"replyAt": "2026-09-09T19:30:05.310Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01L9PAk5ZfhJ53W9YgDDJekn": {
|
||||
"requestAt": "2026-09-09T19:30:29.807Z",
|
||||
"replyAt": "2026-09-09T19:30:31.467Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01APA6Bw9GjU4zMRRzix7bYt": {
|
||||
"requestAt": "2026-09-09T19:30:57.236Z",
|
||||
"replyAt": "2026-09-09T19:30:57.623Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01XnrCrsR4sNA1aVWCroDoKA": {
|
||||
"requestAt": "2026-09-09T19:31:41.556Z",
|
||||
"replyAt": "2026-09-09T19:31:41.899Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01Gc8Bqbok4jzu98v6ocb3LG": {
|
||||
"requestAt": "2026-09-09T19:32:23.137Z",
|
||||
"replyAt": "2026-09-09T19:32:24.184Z",
|
||||
"isError": false
|
||||
},
|
||||
"toolu_01GS78UTYiYA9Y5cXNtwN6on": {
|
||||
"requestAt": "2026-09-09T19:33:02.109Z",
|
||||
"replyAt": "2026-09-09T19:33:02.445Z",
|
||||
"isError": false
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"reviewedTasks": {
|
||||
"writeId": "toolu_01PBxaZDLPnveMvUUmMFJ8Jm",
|
||||
"requestAt": "2026-09-09T19:25:26.587Z",
|
||||
"replyAt": "2026-09-09T19:25:28.211Z",
|
||||
"isError": false,
|
||||
"planSHA256": "cfc5e612add0665b5e7d9c2c154284f99cb23b32ad3d0cc7c99718fe99f3c93a",
|
||||
"lines": [
|
||||
"- [ ] **T1 (P1, human: ~1 day / CC: ~20 min)** \u2014 AuthCache \u2014 Build the facade with constructor DI, per-key write serialization, generation-checked refresh, in-flight dedupe",
|
||||
"- [ ] **T2 (P1, human: ~1 day / CC: ~20 min)** \u2014 AuthCache tests \u2014 Contract suite: key matrix, exact eviction, expired-as-miss, race, dedupe, loader rejection; DI-seam test that services never touch the adapter",
|
||||
"- [ ] **T3 (P1, human: ~1 day / CC: ~20 min)** \u2014 legacyAuthFlow \u2014 Record regression characterization fixtures before any change; assert legacy parity with flag off and external parity with flag on",
|
||||
"- [ ] **T4 (P1, human: ~1 day / CC: ~15 min)** \u2014 Auth entry point \u2014 Per-tenant flag dispatch to legacy or AuthBroker; default off; unknown tenant / flag-store failure \u2192 legacy",
|
||||
"- [ ] **T5 (P1, human: ~4 hr / CC: ~15 min)** \u2014 validate \u2014 Split `validateAndDispatch()` into `validateToken` / `dispatch` / `toAuthError`; typed `AuthError` union; no swallowed catches; single boundary log",
|
||||
"- [ ] **T6 (P2, human: ~1 day / CC: ~20 min)** \u2014 AuthBroker \u2014 Classify the 5 IDP calls; cache discovery/JWKS in AuthCache with TTL + kid-miss refresh; `Promise.all` per-login calls under one deadline; map first rejection via `toAuthError`; add IDP request counter (and limiter if IDP documents a limit)",
|
||||
"- [ ] **T7 (P2, human: ~3 hr / CC: ~10 min)** \u2014 policy \u2014 Implement `evaluatePolicy(claims, tenantPolicy)` as a pure function with typed config; allow/deny/missing/stale-version tests",
|
||||
"- [ ] **T8 (P2, human: ~1 hr / CC: ~5 min)** \u2014 SessionMint \u2014 Constructor takes AuthCache; all writes/invalidations through the facade; tests for logout and suspension invalidation",
|
||||
"- [ ] **T9 (P2, human: ~2 hr / CC: ~10 min)** \u2014 E2E \u2014 Login \u2192 authed request, logout \u2192 rejection, tenant suspension \u2192 rejection, two-tab dedupe, flag flip mid-traffic",
|
||||
"- [ ] **T10 (P3, human: ~30 min / CC: ~5 min)** \u2014 Docs \u2014 Embed the ASCII diagrams listed above as code comments; create TODOS.md with the two approved entries"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"provenance": {
|
||||
"run": "ship-source-ad-full-paid-20260909-v2",
|
||||
"sourceScreen": ".context/ship-source-ad-full-paid-20260909-v2/pty-current-checkpoint-20260909-1929/526d34baa550/terminal.screen.log",
|
||||
"sourceScreenSHA256": "9b38023b26edc962770a3ed1a8c1746838f40aee0fd3d8d55c5ef102c5bde1b0",
|
||||
"actualOutcome": "timeout",
|
||||
"actualAt": "2026-09-09T19:27:05.394Z",
|
||||
"hookState": "not retained after owned cleanup; tests explicitly synthesize valid hook epochs, never historical authorization"
|
||||
},
|
||||
"cwd": "/tmp/gstack-paid-shard-Y3pEVj/tmp/gstack-plan-count-yBLwZl",
|
||||
"config": "/tmp/gstack-paid-shard-Y3pEVj/tmp/gstack-hermetic-1638041-TRh7i1/with-skills/.claude",
|
||||
"sessionId": "e71c124f-4404-4b8b-920c-d672320bce56",
|
||||
"expected": "/tmp/gstack-paid-shard-Y3pEVj/tmp/gstack-e2e-plan-ceo-9iugxu/gstack-test-plan-ceo.md",
|
||||
"events": [
|
||||
{
|
||||
"timestamp": "2026-09-09T19:06:39.489Z",
|
||||
"type": "request",
|
||||
"id": "toolu_01XKdLhbiVr68hDFon5exBBG",
|
||||
"name": "Write",
|
||||
"path": "/tmp/gstack-paid-shard-Y3pEVj/tmp/gstack-e2e-plan-ceo-9iugxu/gstack-test-plan-ceo.md"
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-09-09T19:06:41.730Z",
|
||||
"type": "result",
|
||||
"id": "toolu_01XKdLhbiVr68hDFon5exBBG",
|
||||
"is_error": false
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-09-09T19:08:39.700Z",
|
||||
"type": "request",
|
||||
"id": "toolu_01RJtsV7cugAsWmeMJoLrEx3",
|
||||
"name": "Edit",
|
||||
"path": "/tmp/gstack-paid-shard-Y3pEVj/tmp/gstack-e2e-plan-ceo-9iugxu/gstack-test-plan-ceo.md"
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-09-09T19:08:41.873Z",
|
||||
"type": "result",
|
||||
"id": "toolu_01RJtsV7cugAsWmeMJoLrEx3",
|
||||
"is_error": false
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-09-09T19:09:21.623Z",
|
||||
"type": "request",
|
||||
"id": "toolu_015cxFri1PdUx3jQyCwf2JkJ",
|
||||
"name": "Edit",
|
||||
"path": "/tmp/gstack-paid-shard-Y3pEVj/tmp/gstack-e2e-plan-ceo-9iugxu/gstack-test-plan-ceo.md"
|
||||
}
|
||||
],
|
||||
"screen": " ../gstack-e2e-plan-ceo-9iugxu/gstack-test-plan-ceo.md\n\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\n 185 Rule check: no catch-all rescue anywhere in the remedies; every rescue names\n 186 the mail client's specific classes and logs event id, user id, payment intent\n 187 id, and the attempted operation.\n 188 + \n 189 +**Decision 2.1 (D4) \u2014 APPROVED 2.1A:** send the notification only after the \n 190 +DB transaction commits and dedup records completion; rescue the mail client's \n 191 +specific delivery and timeout exception classes (no `StandardError`); log at \n 192 +warn with event id, user id, payment intent id, operation; return 200. The \n 193 +existing mail failure-rate dashboard/alert and runbook carry the retry. Tests: \n 194 +provider 5xx and timeout each assert user remains paid and response is 200. \n 195 + \n 196 +## Section 3: Security and threat model \n 197 + \n 198 +| # | Threat | Likelihood | Impact | Plan mitigates? | \n 199 +|---|---|---|---|---| \n 200 +| 3.1 | SQL injection through `params.userId` interpolated into a raw fragment. Metadata is merchant-set, but the \n +contract says it is unsanitized; a leaked key, dashboard edit, or upstream bug puts arbitrary text into SQL. | Med\n + | High (read/alter any user row) | NO | \n 201 +| 3.1b | Nonempty malformed `user_id` (\"abc\", 2 KB string, unicode) raises a cast error, wrapper returns 500, Stri\n +pe retries a permanently bad event for 3 days and alerts each time. | Med | Med (alert noise, wasted retries) | NO\n + | \n 202 +| 3.2 | `payment_intent.succeeded` for any amount/currency marks the user paid; no check that the intent matches w\n +hat the user owes. | Low | High (unpaid access) | NO | \n 203 +| 3.3 | Authorization / IDOR: user_id chosen by whoever creates the PaymentIntent (merchant server), not the end u\n +ser; signature guard holds. | Low | High | Yes (retained) | \n 204 +| 3.4 | Replay of a captured webhook | Low | Low | Yes (signature timestamp + dedup) | \n 205 +| 3.5 | PII in logs (email address) | Low | Med | Remedy 2.1 logs user id, not address | \n 206 +| 3.6 | New secrets / dependencies | none | none | n/a | \n 207 +| 3.7 | Audit trail of the payment mutation | n/a | n/a | Yes (traced DB client with event id) | \n\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\n Do you want to make this edit to gstack-test-plan-ceo.md?\n \u276f 1. Yes\n 2. Yes, and switch Line truncated
|
||||
}
|
||||
@@ -88,7 +88,7 @@ function unsupportedSetup(visible: string, question: AskUserQuestionFingerprint,
|
||||
}
|
||||
|
||||
/** Pure routing policy; native packet validation still requires every displayed identity. */
|
||||
function routingSetupActions(question: AskUserQuestionFingerprint, allowTemporarySkip: boolean) {
|
||||
function routingSetupActions(question: AskUserQuestionFingerprint, allowTemporarySkip: boolean, knownSetupOffer = false) {
|
||||
const primary = question.promptSnippet.replace(/^(?:Routing\s*rules|CLAUDE\.md)\s*/i, '').split('?', 1)[0]!;
|
||||
const prompt = primary.replace(/\s+/g, '');
|
||||
const options = question.options.map(option => ({
|
||||
@@ -128,10 +128,57 @@ function routingSetupActions(question: AskUserQuestionFingerprint, allowTemporar
|
||||
// project setup rather than a product routing or taste decision.
|
||||
const claudeTarget = /CLAUDE\.md/i.test(prompt);
|
||||
const quotedPremise = /\b(?:plan|spec|document)\s+(?:quotes?|cites?|references?)\b/i.test(primary);
|
||||
if (!claudeTarget || quotedPremise || (!routingId && !routingPremise)) return null;
|
||||
if (!claudeTarget || quotedPremise || (!knownSetupOffer && !routingId && !routingPremise)) return null;
|
||||
return { add, decline };
|
||||
}
|
||||
|
||||
/** A direct setup offer can carry a decision brief without changing its actions. */
|
||||
function contextualPacketSetup(question: NativePlanQuestion, pending: NativePlanQuestionCall) {
|
||||
if (pending.answered !== false || pending.failed !== false || !pending.sessionId || !pending.toolUseId ||
|
||||
question.options.length !== 2 || question.options.some(option => !option.description?.trim())) return null;
|
||||
const ids = [...question.question.matchAll(/<gstack-qid:[a-z0-9-]+>/gi)];
|
||||
const text = question.question.replace(/<gstack-qid:[a-z0-9-]+>/gi, '').trim();
|
||||
const split = /^([^?]+\?)([\s\S]+)$/.exec(text);
|
||||
if (!split || !split[2]!.trim() || split[2]!.includes('?')) return null;
|
||||
// Strip one presentation label, then match the complete substantive offer.
|
||||
// A quoted/conditional/adjacent offer cannot borrow another tab's actions.
|
||||
const offer = split[1]!.replace(/^D[1-9]\d*\s*[—–:-]\s*/, '').replace(/\s+/g, ' ').trim();
|
||||
const context = [split[2]!.trim(), ...question.options.map(option => option.description!.trim())].join('\n');
|
||||
if (/(?:^|\n|[.!]\s+)(?:Also|Then)\b|\bonly\s+(?:if|after)\b|\bunless\b|\bprovided\s+that\b/i.test(context) ||
|
||||
/\b(?:must|need\s+to|have\s+to)\s+(?:run|complete|finish)\s*\/office-hours\b/i.test(context) ||
|
||||
/(?:\/office-hours|design\s+doc(?:ument)?)\s+(?:is\s+)?(?:required|mandatory)\b/i.test(context) ||
|
||||
/\breview\s+is\s+(?:forbidden|blocked)\b|\b(?:skip|bypass|omit)\s+(?:(?:the|this|full|standard|entire|CEO|design|DX|engineering)\s+)*review\b/i.test(context)) return null;
|
||||
const descriptions = question.options.map(option => option.description!.trim().replace(/^[✅❌]\s*/, ''));
|
||||
const fp: AskUserQuestionFingerprint = { signature: '', observedAtMs: 0, preReview: true,
|
||||
promptSnippet: `${question.header} ${question.question}`,
|
||||
options: question.options.map((option, index) => ({ index: index + 1, label: option.label })) };
|
||||
if (/^Routing(?: rules)?$/i.test(question.header.trim()) &&
|
||||
/^Add\s+(?:gstack\s+)?(?:skill\s+)?routing\s+rules\s+to\s+CLAUDE\.md\?$/i.test(offer) &&
|
||||
(!ids.length || ids.length === 1 && ids[0]![0].toLowerCase() === '<gstack-qid:routing-injection>')) {
|
||||
const actions = routingSetupActions(fp, true, true);
|
||||
if (!actions || actions.add.length !== 1 || actions.decline.length !== 1 ||
|
||||
actions.add[0]!.index === actions.decline[0]!.index) return null;
|
||||
const add = actions.add[0]!.index, decline = actions.decline[0]!.index;
|
||||
if (/^(?:Do not|Don't|Never|Skip|Decline)\s+(?:add(?:ing)?\s+)?routing\s+rules\b/i.test(descriptions[add - 1]!) ||
|
||||
/^Add\s+(?:skill\s+)?routing\s+rules\b/i.test(descriptions[decline - 1]!)) return null;
|
||||
return { kind: 'routing', pick: add };
|
||||
}
|
||||
if (!/^(?:Design doc|Prerequisites?(?: doc)?)$/i.test(question.header.trim()) || ids.length ||
|
||||
!/^Run\s*\/office-hours\s+(?:now|first)(?:\s+for\s+(?:a|the)\s+design\s+doc(?:ument)?)?\?$/i.test(offer)) return null;
|
||||
const run = question.options.findIndex(option => /^Run\s*\/office-hours\s*(?:now|first)(?:\s*\(recommended\))?$/i.test(option.label));
|
||||
const skip = question.options.findIndex(option => /^Skip\s*[—–-]\s*(?:proceed\s+with\s+)?standard\s+review(?:\s*\(recommended\))?$/i.test(option.label));
|
||||
if (run < 0 || skip < 0 || run === skip ||
|
||||
/^(?:Skip|Don't|Do not)\b/i.test(descriptions[run]!) ||
|
||||
/^Run\s*\/office-hours\b/i.test(descriptions[skip]!)) return null;
|
||||
// Corroborate the selected label with its short action clause; the rest
|
||||
// of the description may explain tradeoffs without changing that action.
|
||||
const sentence = /^([^.!?]+)([.!?]|$)/.exec(descriptions[skip]!);
|
||||
const action = sentence?.[1]?.trim() ?? '';
|
||||
if (sentence?.[2] === '?' || !/^(?:Review\s+(?:starts?|begins?)\s+(?:immediately|now)|(?:Start|Begin)\s+(?:the\s+)?(?:standard\s+)?review\s+(?:immediately|now)|Proceed\s+(?:directly\s+)?with\s+(?:the\s+)?standard\s+review)(?:\s+(?:using|with|on)\s+[^.!?]+)?$/i.test(action) ||
|
||||
/\b(?:after|when|once|until|if|unless|provided|not|never)\b|n['’]t\b/i.test(action)) return null;
|
||||
return { kind: 'prerequisite', pick: skip + 1 };
|
||||
}
|
||||
|
||||
/** Numbered setup wording may vary; newly admitted forms still consume every description. */
|
||||
function numberedPacketSetup(question: NativePlanQuestion, pending: NativePlanQuestionCall) {
|
||||
if (pending.answered !== false || pending.failed !== false || !pending.sessionId || !pending.toolUseId || question.options.length !== 2) return null;
|
||||
@@ -189,7 +236,8 @@ function setupPacketDecision(visible: string, seen: ReadonlySet<string>, pending
|
||||
// Require an actual prerequisite offer, not a product question that
|
||||
// happens to mention the absence of an office-hours design document.
|
||||
const offer = /^No\s+design\s+doc\s+(?:found|exists)(?:\s+for\s+(?:this|the)\s+(?:branch|project))?\.\s*(?:\/office-hours\s+(?:produces|creates|provides)\s+(?:a\s+)?(?:structured\s+)?(?:design\s+doc(?:ument)?|problem\s+statement)(?:,?\s+(?:and\s+)?(?:premise\s+challenge|(?:explored\s+)?alternatives))*(?:\s*[—–-]\s*(?:sharper|better)\s+input\s+for\s+(?:the|this)\s+review)?\.\s*)?(?:Want\s+to\s+|Would\s+you\s+like\s+to\s+)?Run\s+(?:it|\/office-hours)\s+(?:now|first)(?:\s+or\s+proceed\s+with\s+standard\s+review)?\s*\?$/i.test(offerText);
|
||||
return prerequisite !== null && run.length === 1 && offer ? { kind: 'prerequisite', pick: prerequisite } : numberedPacketSetup(question, pending);
|
||||
return prerequisite !== null && run.length === 1 && offer ? { kind: 'prerequisite', pick: prerequisite }
|
||||
: numberedPacketSetup(question, pending) ?? contextualPacketSetup(question, pending);
|
||||
});
|
||||
if (policies.some(policy => !policy) || new Set(policies.map(policy => policy!.kind)).size !== 2) return waiting;
|
||||
|
||||
|
||||
@@ -360,10 +360,107 @@ function manualHandoffIndex(fp: AskUserQuestionFingerprint): number | null {
|
||||
return manual.findIndex(Boolean) + 1;
|
||||
}
|
||||
|
||||
/** Every offered explanation must remain a clause about this review handoff. */
|
||||
function closedNextReviewExplanations(question: string, descriptions: string[]): boolean {
|
||||
// Validate each complete sentence/line, rather than discarding prose under
|
||||
// an accepted heading. A new imperative has no navigation subject and
|
||||
// cannot borrow the preceding sentence's administrative classification.
|
||||
const navigation = [
|
||||
/^(?:The )?CEO review (?:is (?:done|complete|cleared)(?: and clears scope and strategy)?|cleared scope and strengthened both test assertions)$/i,
|
||||
/^The engineering review is the one gate that must pass before shipping \(skip_eng_review is false\)$/i,
|
||||
/^It checks architecture, code quality, and test design in depth$/i,
|
||||
/^gstack['’]s shipping gate is the eng review, which checks architecture and test design$/i,
|
||||
/^it has not run for this plan yet$/i,
|
||||
/^(?:There is no UI|No UI scope was found), so a design review does not apply$/i,
|
||||
/^Skipping (?:the )?eng review leaves the (?:required gate unmet, so the readiness dashboard stays NOT CLEARED until someone runs it later|ship dashboard NOT CLEARED)$/i,
|
||||
/^running it costs a few minutes on a two-test plan$/i,
|
||||
/^[A-Z] because eng review is the required (?:shipping gate and this plan is now precise enough for it to run quickly|gate and the plan changed since it was written \(two assertions strengthened\), so the tests deserve a second read)$/i,
|
||||
/^options differ in kind(?: \(which workflow runs next\))?, not coverage [—–-] no completeness score$/i,
|
||||
/^clear the (?:required )?gate now versus (?:handling reviews on your own schedule|implement first and review later)$/i,
|
||||
/^Clears the required (?:engineering gate while the plan and its two approved remedies are fresh|shipping gate on the review readiness dashboard)$/i,
|
||||
/^A second structured pass over the test design catches anything the scope review did not$/i,
|
||||
/^One more interactive review session before implementation starts$/i,
|
||||
/^Ends the review chain here$/i,
|
||||
/^you decide when the eng review runs$/i,
|
||||
/^No further (?:questions this session|review prompts in this session)$/i,
|
||||
/^(?:The required eng gate stays unmet and the dashboard remains NOT CLEARED|Dashboard stays NOT CLEARED until an eng review runs)$/i,
|
||||
/^Second read of the exact assertions and the await-then-count ordering before code is written$/i,
|
||||
/^A few extra minutes on a plan that is already two tests against existing probes$/i,
|
||||
/^Move straight to implementing T[1-9]\d* and T[1-9]\d* now$/i,
|
||||
/^Start the eng review against the updated plan after this review exits$/i,
|
||||
/^End here$/i,
|
||||
/^run reviews yourself later$/i,
|
||||
];
|
||||
const duration = String.raw`~?\d+(?:\.\d+)?\s*(?:minutes?|mins?|hours?|hrs?|days?|weeks?)`;
|
||||
const timing = new RegExp(String.raw`\s*\(human:\s*${duration}\s*/\s*CC:\s*${duration}\)$`, 'i');
|
||||
let metadata = 0;
|
||||
const body = question.split('\n').slice(1).concat(descriptions.flatMap(text => text.split('\n')));
|
||||
for (const raw of body) {
|
||||
const line = raw.trim();
|
||||
if (!line) continue;
|
||||
if (/^Project\/branch\/task:/.test(line)) {
|
||||
// Only the project/mode recap is metadata, never a repair paragraph.
|
||||
if (++metadata !== 1 || !/^Project\/branch\/task: (?:[\w-]+ on [\w/-]+, \/plan-ceo-review \(HOLD SCOPE\) finished on the payment test-coverage plan|`[\w/-]+`, CEO review of PLAN\.md complete \(HOLD SCOPE, 0 critical gaps, [1-9]\d* assertion fixes approved\))\.$/.test(line)) return false;
|
||||
continue;
|
||||
}
|
||||
if (line === 'Pros / cons:') continue;
|
||||
if (/^[A-Z][):] /.test(line)) {
|
||||
const offered = line.replace(/^[A-Z][):] /, '').replace(timing, '').replace(/ \(recommended\)$/, '');
|
||||
if (!/^(?:Run \/plan-eng-review next|Skip, handle reviews manually)$/.test(offered)) return false;
|
||||
continue;
|
||||
}
|
||||
const prose = line.replace(/^(?:ELI10|Stakes if we pick wrong|Recommendation|Note|Net):\s*/, '')
|
||||
.replace(/^[✅❌]\s*/, '').replace(timing, '');
|
||||
const clauses = prose.split(/[.;]\s+|[.]$/).map(s => s.trim()).filter(Boolean);
|
||||
if (!clauses.length || !clauses.every(clause => navigation.some(pattern => pattern.test(clause)))) return false;
|
||||
}
|
||||
return metadata === 1;
|
||||
}
|
||||
|
||||
/** Evidence-only next-review accounting; this never selects a pending option. */
|
||||
function completedNextReviewBrief(fp: AskUserQuestionFingerprint): boolean {
|
||||
const call = fp.nativeCall;
|
||||
if (!call?.sessionId || !call.toolUseId || call.answered !== true || call.failed !== false ||
|
||||
fp.signature !== `${call.sessionId}:${call.toolUseId}` || call.questions.length !== 1 ||
|
||||
(fp.nativeQuestionIndex !== undefined && fp.nativeQuestionIndex !== 0) ||
|
||||
!Array.isArray(call.unansweredQuestionIndices) || call.unansweredQuestionIndices.length ||
|
||||
Object.keys(call.answers ?? {}).length !== 1 || !Number.isFinite(Date.parse(call.answeredAt ?? ''))) return false;
|
||||
const q = call.questions[0]!;
|
||||
if (q.multiSelect || !/^Next (?:step|review)$/i.test(q.header.trim()) || q.options.length !== 2 ||
|
||||
fp.options.length !== 2 || !fp.options.every((o, i) => o.index === i + 1 && o.label === q.options[i]!.label) ||
|
||||
!q.options.some(o => o.label === call.answers?.[q.question]) || /<gstack-qid/i.test(q.question)) return false;
|
||||
const question = q.question.trim().replace(/^D\d+\s*[—–-]\s*/i, '');
|
||||
const lines = question.split('\n').map(line => line.trim()).filter(Boolean);
|
||||
// Numbered headings and echoed pros/cons are presentation. Require the
|
||||
// actual navigation query, explicit current CEO closure, and a final brief
|
||||
// boundary; a new question or directive after that boundary stays work.
|
||||
if (!/^(?:CEO review (?:is )?(?:complete|done|cleared)\. )?Which review runs next\?$/i.test(lines[0]!) ||
|
||||
!/^Net:\s+[^\n]+[.!]$/.test(lines.at(-1) ?? '') ||
|
||||
!/(?:^|[.!?]\s+|^ELI10:\s*)(?:The\s+)?CEO\s+review\s+(?:is\s+)?(?:complete|done|cleared)\b/im.test(question)) return false;
|
||||
const labels = q.options.map(o => o.label.trim().replace(/^[A-Z][).:]\s*/i, '').replace(/\s*\(recommended\)\s*$/i, ''));
|
||||
if (labels.filter(label => /^Run \/plan-eng-review next$/i.test(label)).length !== 1 ||
|
||||
labels.filter(label => /^Skip\s*[,—–-]\s*(?:(?:I['’]ll\s+)?handle reviews manually|manual reviews)$/i.test(label)).length !== 1) return false;
|
||||
const context = [question, ...q.options.map(o => o.description ?? '')].join('\n')
|
||||
.replace(/^Stakes if we pick wrong:/m, 'Stakes:');
|
||||
// Only the next Eng gate can keep the readiness dashboard uncleared.
|
||||
// Its temporal explanation is not a condition on current CEO closure;
|
||||
// every other unfinished-work and conditional-closure guard still applies.
|
||||
const navigationContext = context.replace(
|
||||
/\b((?:(?:readiness|ship)\s+)?dashboard\s+(?:stays|remains)\s+NOT\s+CLEARED)\s+until\s+(?:someone\s+runs\s+it|(?:an?|the)\s+eng(?:ineering)?\s+review\s+runs)(?:\s+later)?(?=[.!]|\n|$)/gi,
|
||||
'$1',
|
||||
);
|
||||
return q.options.every(o => o.description?.trim()) &&
|
||||
closedNextReviewExplanations(question, q.options.map(o => o.description ?? '')) &&
|
||||
!/`{3}|~{3}|(?:^|\n)\s*>|\b(?:example|quoted source)\s*:/im.test(context) &&
|
||||
!/\bCEO\s+review\b[^.!?\n]{0,80}\b(?:not|never|incomplete|unfinished)\b/i.test(context) &&
|
||||
/\b(?:Eng|engineering) review\b[^.!?]{0,180}\bgate\b/i.test(context) && closedNavigationContext(navigationContext);
|
||||
}
|
||||
|
||||
/** Classification happens only after one real, successful, fully answered native call. */
|
||||
export function isCeoCompletionHandoff(fp: AskUserQuestionFingerprint): boolean {
|
||||
const call = fp.nativeCall;
|
||||
if (!call?.answered || call.failed || !Array.isArray(call.unansweredQuestionIndices) || call.unansweredQuestionIndices.length) return false;
|
||||
if (completedNextReviewBrief(fp)) return true;
|
||||
if (manualHandoffIndex(fp) === null) return false;
|
||||
const q = call.questions[0]!;
|
||||
// A free-form answer can introduce a new substantive request. Do not
|
||||
|
||||
@@ -164,9 +164,19 @@ export function hasStaleFillRaceFinding(report: string): boolean {
|
||||
|
||||
function hasProseStaleFillFinding(report: string): boolean {
|
||||
// Copied source, diagrams and quoted examples cannot supply a finding.
|
||||
const prose = report.replace(/```[\s\S]*?```/g, '').replace(/^\s*>.*$/gm, '');
|
||||
let fence: { char: string; length: number } | null = null;
|
||||
const prose = report.split('\n').map(line => {
|
||||
const delimiter = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line);
|
||||
if (delimiter) {
|
||||
const run = delimiter[1]!;
|
||||
if (!fence) fence = { char: run[0]!, length: run.length };
|
||||
else if (run[0] === fence.char && run.length >= fence.length && !delimiter[2]!.trim()) fence = null;
|
||||
return '';
|
||||
}
|
||||
return fence || /^\s*>/.test(line) || /^(?: {4}|\t)/.test(line) ? '' : line;
|
||||
}).join('\n');
|
||||
// Independent list items and table rows cannot borrow each other's words.
|
||||
const blocks = prose.split(/\n\s*\n|\n(?=\s*(?:\||\d+\.\s|[-*]\s))/).map(block => block.trim());
|
||||
const blocks = prose.split(/\n\s*\n|\n(?=\s*(?:#{1,6}\s|\||\d+\.\s|[-*]\s))/).map(block => block.trim());
|
||||
const normalize = (text: string) => text.replace(/[*_`]/g, '').replace(/\s+/g, ' ').trim();
|
||||
return blocks.some((block, index) => {
|
||||
const text = normalize(block);
|
||||
@@ -177,7 +187,15 @@ function hasProseStaleFillFinding(report: string): boolean {
|
||||
const fill = fillPattern.test(text);
|
||||
const invalidation = /\b(?:invalidat|evict|write|commit|delet)\w*\b/i.test(text);
|
||||
const ordering = /\b(?:after|later|resum\w*)\b|out[- ]of[- ]order/i.test(text);
|
||||
if (!stale || !inFlight || !read || !fill || !invalidation || !ordering) return false;
|
||||
// A review may identify the ordering defect directly as missing coordination
|
||||
// between cache fills and writes that violates read-after-write freshness.
|
||||
// That is independent evidence even when the old-value trace is a diagram.
|
||||
const premise = /(?:^|[.;]\s+)(?:\[Amended:[^\]]{1,80}\]\s*)?(?:the\s+)?(?:original|current|proposed)\s+(?:sketch|wrapper|implementation)\s+(?:had|has)\s+no\s+coordination\s+between\s+(?:an?\s+)?cache\s+fill\s+and\s+(?:an?\s+)?write\b/i.exec(text);
|
||||
const conclusion = /(?:^|[.;]\s+)(?:finding\s+[\w.-]+\s+(?:showed|shows)\s+)?(?:this|that|it)\s+(?:violates|breaks)\s+the\s+read[- ]after[- ]write\s+(?:rule|contract|guarantee|invariant)(?:[.!](?=\s|$)|$)/i.exec(text);
|
||||
const coordinationGap = premise !== null && conclusion !== null && premise.index < conclusion.index
|
||||
&& !/["“”]|\b(?:if|example|template|quoted)\b/i.test(text)
|
||||
&& !/\b(?:example|template|source|quoted|format)\b[^.]*:\s*$/i.test(blocks[index - 1] ?? '');
|
||||
if ((!stale || !inFlight || !read || !fill || !invalidation || !ordering) && !coordinationGap) return false;
|
||||
|
||||
// A neighboring explanation/remedy belongs to this paragraph only until
|
||||
// another named finding/section/table row begins. In particular, a
|
||||
|
||||
@@ -1584,6 +1584,17 @@ export function planCountPrerequisitePick(fp: AskUserQuestionFingerprint, active
|
||||
const nativeSkip = q.options.findIndex(option =>
|
||||
/^Skip(?:\s*[—–-]\s*proceed)?(?:\s*\(recommended\))?$/i.test(option.label) &&
|
||||
/^(?:(?:The\s+)?plan\s+(?:scope\s+)?is\s+(?:already\s+)?(?:precise|clear|well-defined|explicit)\.\s*)?Proceed\s+(?:with\s+standard(?:\s+DX(?:\s+(?:POLISH|EXPANSION|TRIAGE))?)?\s+review|straight\s+to\s+Step\s*0\s+premise\s+challenge\s+and\s+approach\s+alternatives)\.?$/i.test((option.description ?? '').trim()));
|
||||
// A full Skip label can use a comma. Admit that form only from the
|
||||
// bound active tab's direct opposed offer and unconditional review action.
|
||||
const offer = q.question.split('?', 1)[0]!.replace(/^D[1-9]\d*\s*[—–:-]\s*/, '').trim();
|
||||
const commaSkip = q.options.findIndex(option =>
|
||||
/^Skip\s*,\s*(?:proceed\s+with\s+)?standard\s+review(?:\s*\(recommended\))?$/i.test(option.label) &&
|
||||
/^Proceed\s+(?:(?:directly|straight)\s+)?(?:with\s+(?:the\s+)?standard\s+review|to\s+Step\s*0\s+of\s+(?:the\s+)?(?:CEO\s+)?review)\.?$/i.test((option.description ?? '').trim()));
|
||||
if (nativeRun >= 0 && commaSkip >= 0 && nativeRun !== commaSkip && call.sessionId && call.toolUseId &&
|
||||
q.question.split('?').length === 2 &&
|
||||
/^Run\s*\/office-hours\s+(?:now|first),?\s+or\s+proceed\s+with\s+(?:the\s+)?standard\s+review$/i.test(offer) &&
|
||||
/^(?:Build|Create|Produce)\s+(?:a|the)\s+design\s+doc(?:ument)?\s+first[,;]\s*then\s+resume\s+(?:the|standard|CEO)\s+review\.?$/i.test((q.options[nativeRun]!.description ?? '').trim()) &&
|
||||
!/\b(?:must|need\s+to|have\s+to)\s+(?:run|complete|finish)\s*\/office-hours\b|(?:\/office-hours|design\s+doc(?:ument)?)\s+(?:is\s+)?(?:required|mandatory)\b|\breview\s+is\s+(?:forbidden|blocked)\b/i.test(q.question)) return commaSkip + 1;
|
||||
if (nativeRun >= 0 && nativeSkip >= 0 && nativeRun !== nativeSkip) return nativeSkip + 1;
|
||||
}
|
||||
}
|
||||
@@ -2152,10 +2163,19 @@ function qidlessCeoFinding(fp: AskUserQuestionFingerprint): boolean {
|
||||
new Set(q.options.map(option => option.label)).size !== q.options.length ||
|
||||
!q.options.some(option => option.label === call.answers?.[q.question])) return false;
|
||||
const title = q.question.split('\n')[0]!;
|
||||
const finding = /^(?:D[1-9]\d*\s*[—–-]\s*)?Finding\s+([1-9]\d*)(?:\s+\(Section\s+[1-9]\d*\))?\s*:\s*[^\n?]+\?$/i.exec(title);
|
||||
if (finding) return !/\(Section\s/i.test(title) || q.header.trim().toLowerCase() === `finding ${finding[1]}`;
|
||||
const issue = /^D[1-9]\d*\s+\(issue\s+([1-9]\d*(?:\.[1-9]\d*)*)\)\s*[—–-]\s*[^\n?]+\?$/i.exec(title);
|
||||
return Boolean(issue && q.header.trim().toLowerCase() === `issue ${issue[1]}`);
|
||||
// The issue identity is separate from the decision counter and section
|
||||
// numbering. A completed "Issue 2" choice and "Finding 2.1" choice carry
|
||||
// the same review evidence as the already-supported numbered findings.
|
||||
const normalized = title.replace(/^D\d+\s*[—–-]\s*/i, '');
|
||||
const identity = /^(Finding|Issue)\s+([1-9]\d*(?:\.[1-9]\d*)*)(?:\s+\(Section\s+[1-9]\d*\))?\s*:\s*[^\n?]+\?$/i.exec(normalized);
|
||||
const parenthesized = /^D[1-9]\d*\s+\(issue\s+([1-9]\d*(?:\.[1-9]\d*)*)\)\s*[—–-]\s*[^\n?]+\?$/i.exec(title);
|
||||
if (!identity && !parenthesized) return false;
|
||||
const expected = identity ? `${identity[1]} ${identity[2]}`.toLowerCase() : `issue ${parenthesized![1]}`;
|
||||
const header = q.header.trim().toLowerCase();
|
||||
const numberedHeader = /^(?:finding|issue)\s+[1-9]\d*(?:\.[1-9]\d*)*$/.test(header);
|
||||
// A descriptive header is fine; a supplied identity must not contradict
|
||||
// the finding. Preserve the original section/parenthesis binding guards.
|
||||
return !(numberedHeader || parenthesized || /\(Section\s/i.test(title)) || header === expected;
|
||||
}
|
||||
|
||||
export const ceoFirstReviewAUQ: Step0BoundaryPredicate = (fp) =>
|
||||
@@ -2367,9 +2387,53 @@ function engSharedMutableCacheAUQ(fp: AskUserQuestionFingerprint): boolean {
|
||||
core(accepted.description ?? '') === 'Note the shared-global pattern in the review report as a known risk. Leave the plan unchanged. Suitable only if the runtime is single-threaded and concurrent mutation is architecturally impossible.');
|
||||
}
|
||||
|
||||
/** A completed explicit issue retains its identity when question tuning omits qids. */
|
||||
function engNumberedFindingAUQ(fp: AskUserQuestionFingerprint): boolean {
|
||||
const call = fp.nativeCall;
|
||||
if (!call?.sessionId || !call.toolUseId || call.answered !== true || call.failed !== false ||
|
||||
fp.signature !== `${call.sessionId}:${call.toolUseId}` || call.questions.length !== 1 ||
|
||||
!Array.isArray(call.unansweredQuestionIndices) || call.unansweredQuestionIndices.length ||
|
||||
Object.keys(call.answers ?? {}).length !== 1 ||
|
||||
(fp.nativeQuestionIndex !== undefined && fp.nativeQuestionIndex !== 0) ||
|
||||
!Number.isFinite(Date.parse(call.answeredAt ?? ''))) return false;
|
||||
const q = call.questions[0]!;
|
||||
if (q.multiSelect || q.options.length < 2 || q.options.length > 4 ||
|
||||
new Set(q.options.map(o => o.label)).size !== q.options.length ||
|
||||
q.options.filter(o => o.label === call.answers?.[q.question]).length !== 1 ||
|
||||
fp.options.length !== q.options.length || !fp.options.every((o, i) => o.index === i + 1 && o.label === q.options[i]!.label) ||
|
||||
/<gstack-qid/i.test(q.question)) return false;
|
||||
const title = q.question.split('\n')[0]!.replace(/^D[1-9]\d*\s*[—–:-]\s*/i, '');
|
||||
const issue = /^(?:Issue|Finding)\s+([1-9]\d*(?:\.[1-9]\d*)*)(?:\s*\(D[1-9]\d*\))?\s*[—–:-]\s*([^\n?]+\?)$/i.exec(title);
|
||||
const header = /^(?:Arch(?:itecture)?|Code\s+Q(?:uality)?|Tests?|Perf(?:ormance)?)\s+([1-9]\d*(?:\.[1-9]\d*)*)$/i.exec(q.header.trim());
|
||||
if (!issue || !header || issue[1] !== header[1]) return false;
|
||||
// A title number or verb can also label report administration. Keep the
|
||||
// concrete defect, implementation query and offered technical alternatives
|
||||
// together. Unknown issue families remain on the existing qid paths.
|
||||
const label = (s: string) => s.trim().replace(/^[1-9]\d*[A-Z]\)\s*/i, '').replace(/\s*\(recommended\)$/i, '');
|
||||
const alternatives = (a: RegExp, b: RegExp) => [a, b].every(pattern =>
|
||||
q.options.some(option => pattern.test(label(option.label)) && Boolean(option.description?.trim())));
|
||||
const body = issue[2]!;
|
||||
return (
|
||||
/^[A-Za-z_$][\w$]* is a (?:global|shared) mutable module-level export that (?:two|multiple|\d+) services mutate\. Inject it(?: instead)?\?$/i.test(body) &&
|
||||
alternatives(/^Constructor[- ]inject$/i, /^Getter \+ reset hook$/i)
|
||||
) || (
|
||||
/^Two writers, no serialization: an? [A-Za-z_$][\w$]* write can land after an? [A-Za-z_$][\w$]* invalidation and resurrect a revoked token\. (?:Guard it|Serialize the writes)\?$/i.test(body) &&
|
||||
alternatives(/^Invalidation epoch in [A-Za-z_$][\w$]*$/i, /^Re-check before write$/i)
|
||||
) || (
|
||||
/^[A-Za-z_$][\w$]*\(\) is \d+ lines with (?:two|three|multiple|\d+) nested try\/catch blocks that each swallow a different error class\. Restructure it\?$/i.test(body) &&
|
||||
alternatives(/^Split \+ typed Result$/i, /^Flatten \+ log$/i)
|
||||
) || (
|
||||
/^Planned coverage is unit \+ integration on the new components only\. Add an? end-to-end [\w -]+ test across (?:two|multiple|\d+) tenants\?$/i.test(body) &&
|
||||
alternatives(/^Add E2E journey test$/i, /^Facade isolation test only$/i)
|
||||
) || (
|
||||
/^Token validation makes \d+ sequential [A-Za-z_$][\w$]* calls that are independent\. Parallelize, and with what failure semantics\?$/i.test(body) &&
|
||||
alternatives(/^Promise\.all \+ timeouts \+ typed errors$/i, /^Bare Promise\.all$/i)
|
||||
);
|
||||
}
|
||||
|
||||
/** An answered substantive finding can start review even in a mixed setup packet. */
|
||||
export const engFirstReviewAUQ: Step0BoundaryPredicate = (fp) => {
|
||||
if (engExplicitRepairAUQ(fp) || engArchitectureChoiceAUQ(fp) || engDependencyBindingAUQ(fp) || engSharedMutableCacheAUQ(fp)) return true;
|
||||
if (engNumberedFindingAUQ(fp) || engExplicitRepairAUQ(fp) || engArchitectureChoiceAUQ(fp) || engDependencyBindingAUQ(fp) || engSharedMutableCacheAUQ(fp)) return true;
|
||||
const call = fp.nativeCall;
|
||||
if (!call?.answered || call.failed) return false;
|
||||
return call.questions.some(q => {
|
||||
@@ -3203,7 +3267,7 @@ export async function runPlanSkillCounting(opts: {
|
||||
/** Accepted artifact rendering is not a finding; its answer still requires a fresh report. */
|
||||
isArtifactGenerationAUQ?: Step0BoundaryPredicate;
|
||||
/** Optional issue classifier across phases; receives full native call metadata. */
|
||||
isReviewAUQ?: (fp: AskUserQuestionFingerprint) => boolean;
|
||||
isReviewAUQ?: (fp: AskUserQuestionFingerprint, priorCalls?: readonly NativePlanQuestionCall[]) => boolean;
|
||||
/** Narrow caller-specific selection; null retains the normal answer policy.
|
||||
* The first argument retains full pending metadata for existing callers.
|
||||
* Native-bound selection uses activeCapture, whose metadata is present only
|
||||
@@ -3354,7 +3418,7 @@ export async function runPlanSkillCounting(opts: {
|
||||
if (transcript.status === 'error') {
|
||||
return snapshot('transcript_unavailable', transcript.error!, visible);
|
||||
}
|
||||
for (const call of transcript.calls) {
|
||||
for (const [callIndex, call] of transcript.calls.entries()) {
|
||||
const signature = `${call.sessionId}:${call.toolUseId}`;
|
||||
if (!call.answered || countedCalls.has(signature)) continue;
|
||||
const fp = nativePlanCallFingerprint(call, Date.now() - startedAt, !boundaryFired);
|
||||
@@ -3364,7 +3428,7 @@ export async function runPlanSkillCounting(opts: {
|
||||
fp.administrative = phase.administrative;
|
||||
administrativeCount += 1;
|
||||
} else {
|
||||
fp.preReview = opts.isReviewAUQ ? !opts.isReviewAUQ(fp) : phase.preReview;
|
||||
fp.preReview = opts.isReviewAUQ ? !opts.isReviewAUQ(fp, transcript.calls.slice(0, callIndex)) : phase.preReview;
|
||||
if (fp.preReview) step0Count += 1;
|
||||
else reviewCount += 1;
|
||||
}
|
||||
|
||||
@@ -54,12 +54,34 @@ function numberedVisualHierarchyFinding(fp: AskUserQuestionFingerprint): boolean
|
||||
/^Leave the gap named but unresolved\. Engineer decides the button styles at implementation time without a spec\. Risk: inconsistency with the design system or re-work after review\.$/i.test(q.options[defer]!.description?.trim() ?? '');
|
||||
}
|
||||
|
||||
/** A qidless Issue with its own design gap is a finding, independent of D numbering. */
|
||||
function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean {
|
||||
const call = fp.nativeCall;
|
||||
if (!call || call.answered !== true || call.failed !== false || !call.sessionId || !call.toolUseId ||
|
||||
call.questions.length !== 1 || !Array.isArray(call.unansweredQuestionIndices) || call.unansweredQuestionIndices.length ||
|
||||
fp.signature !== `${call.sessionId}:${call.toolUseId}` ||
|
||||
(fp.nativeQuestionIndex !== undefined && fp.nativeQuestionIndex !== 0)) return false;
|
||||
const q = call.questions[0]!;
|
||||
const title = q.question.split('\n')[0]!.trim();
|
||||
const issue = /^(?:D[1-9]\d*\s*[—–:-]\s*)?Issue ([1-9]\d*)(?: \((?:G[1-9]\d*, )?(?:Visual Hierarchy|Spacing|Color|Typography|Motion)\))?: ([^?]+)\?$/i.exec(title);
|
||||
if (!issue || !new RegExp(`^Issue ${issue[1]}$`, 'i').test(q.header.trim()) ||
|
||||
!/\b(?:fix|resolve|address)\b/i.test(title) || /<gstack-qid:/i.test(q.question) || q.multiSelect ||
|
||||
q.options.length < 2 || new Set(q.options.map(o => o.label)).size !== q.options.length ||
|
||||
fp.options.length !== q.options.length || !fp.options.every((o, i) => o.index === i + 1 && o.label === q.options[i]!.label) ||
|
||||
!q.options.some(o => o.label === call.answers?.[q.question])) return false;
|
||||
// The numbered headline must ask about a concrete design requirement.
|
||||
// Reviewer participation or workflow navigation can also use Issue labels.
|
||||
if (!/\b(?:buttons?|hierarchy|spacing|contrast|colou?rs?|labels?|typography|fonts?|loading|spinner|skeleton|motion)\b/i.test(issue[2]!)) return false;
|
||||
return q.options.some(o => /^(?:[1-9]\d*[A-Z][).]\s*)?(?:Defer|Leave|Keep|Accept the gap)\b/i.test(o.label)) &&
|
||||
q.options.some(o => /\b(?:closing|closes|fixes|resolves?|applies?)\b/i.test(o.description ?? ''));
|
||||
}
|
||||
|
||||
/** A completed finding can start the passes when the caller already supplied the focus. */
|
||||
export function isDesignCountFirstReview(fp: AskUserQuestionFingerprint): boolean {
|
||||
const call = fp.nativeCall;
|
||||
if (!call?.answered || call.failed) return false;
|
||||
if (isDesignCountSetup(fp)) return false;
|
||||
if (numberedVisualHierarchyFinding(fp)) return true;
|
||||
if (numberedVisualHierarchyFinding(fp) || ordinaryDesignIssue(fp)) return true;
|
||||
if (designFirstReviewAUQ(fp)) return true;
|
||||
return call.questions.some(q => {
|
||||
if (!call.answers?.[q.question] || q.options.length < 2) return false;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { AskUserQuestionFingerprint } from './claude-pty-runner';
|
||||
import type { NativePlanQuestionCall } from './plan-count-transcript';
|
||||
|
||||
/** Concrete product decisions, separate from the skill's mandatory Step-0 confirmations. */
|
||||
export const DEVEX_COUNT_FILES: Record<string, string> = {
|
||||
@@ -120,6 +121,8 @@ module and its sample data are included and work as documented.
|
||||
export function planDevexCountFixture(planPath: string): string {
|
||||
return [
|
||||
`Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
|
||||
'This is an interactive review; a responder is available for AskUserQuestion.',
|
||||
'Present each unresolved decision and wait for its answer.',
|
||||
'',
|
||||
'# Plan: EvalKit SDK beta release polish',
|
||||
'',
|
||||
@@ -428,9 +431,174 @@ function answeredContractRepair(fp: AskUserQuestionFingerprint): boolean {
|
||||
return absentPackageFile || conflictingGate || unshippedQuickstart || unreachableBenchmark;
|
||||
}
|
||||
|
||||
/** An explicitly quoted developer account plus accuracy-only choices adds no repair. */
|
||||
function answeredQuotedAccuracy(fp: AskUserQuestionFingerprint): boolean {
|
||||
const call = fp.nativeCall;
|
||||
if (!call?.sessionId || !call.toolUseId || call.answered !== true || call.failed !== false ||
|
||||
call.questions.length !== 1 || fp.signature !== `${call.sessionId}:${call.toolUseId}` ||
|
||||
!Array.isArray(call.unansweredQuestionIndices) || call.unansweredQuestionIndices.length ||
|
||||
Object.keys(call.answers ?? {}).length !== 1 || !Number.isFinite(Date.parse(call.answeredAt ?? ''))) return false;
|
||||
const q = call.questions[0]!;
|
||||
if (q.header !== 'Narrative' || q.multiSelect || q.options.length !== 3 || fp.options.length !== 3 ||
|
||||
!fp.options.every((o,i) => o.index === i+1 && o.label === q.options[i]!.label) ||
|
||||
!q.options.some(o => call.answers?.[q.question] === o.label) || /<gstack-qid/i.test(q.question)) return false;
|
||||
const expectedOptions = [
|
||||
['This is accurate, proceed', 'Use this narrative as the Developer Perspective section and continue to friction-point decisions.'],
|
||||
['Some of this is wrong, let me correct it', 'Tell me which steps differ; I will fold corrections in before scoring.'],
|
||||
['This is way off, the actual experience is...', 'Describe the real flow and I will rebuild the narrative from it.'],
|
||||
];
|
||||
if (!q.options.every((o,i) => o.label.replace(/ \(recommended\)$/i, '') === expectedOptions[i]![0] && o.description === expectedOptions[i]![1])) return false;
|
||||
const parts = q.question.replace(/^D\d+\s*[—–-]\s*/, '').split(/\n\s*\n/);
|
||||
if (parts.length < 5 || parts[0] !== 'Empathy narrative: does this match what your ML engineer experiences today?' ||
|
||||
!/^Project\/branch\/task: [\w/-]+ branch, [\w. -]+ beta polish, tracing the README getting-started path as written\.$/.test(parts[1]!) ||
|
||||
parts[2] !== 'Here is what I think your ML engineer experiences today:') return false;
|
||||
const quoted = parts.slice(3,-1).join('\n\n');
|
||||
// These are source words in an explicitly bounded quotation, not approval
|
||||
// of any action they mention. No unquoted paragraph may intervene.
|
||||
if (!/^"I [\s\S]+"$/.test(quoted) || (quoted.match(/"/g)?.length ?? 0) !== 2) return false;
|
||||
const explanatory = [
|
||||
"ELI10: This narrative becomes the 'Developer Perspective' section the implementer reads. If it is wrong, the whole review is calibrated against a fake developer.",
|
||||
'Stakes if we pick wrong: we fix friction your developer never hits, or miss the one that actually loses them.',
|
||||
'Recommendation: A because every step above quotes a documented contract in README.md, docs/api.md, docs/current-contracts.md, or docs/package-contents.txt rather than a guess.',
|
||||
'Note: options differ in kind, not coverage — no completeness score.',
|
||||
'A) This is accurate, proceed with this understanding (recommended)',
|
||||
'✅ Every friction point is grounded in a specific documented line, not hypothesized',
|
||||
'✅ Lets the review move straight to per-friction-point decisions with shared context',
|
||||
'❌ If the docs lag the real runtime, a fixed contract could be reviewed as if still broken',
|
||||
'B) Some of this is wrong, let me correct it',
|
||||
'✅ Corrections get folded into the narrative before any scoring happens',
|
||||
'✅ Catches doc-versus-runtime drift the repo cannot show me',
|
||||
'❌ Requires you to spell out which steps differ and how',
|
||||
'C) This is way off, the actual experience is...',
|
||||
'✅ Resets the review against your real onboarding flow',
|
||||
'✅ Prevents scoring against contracts that no longer exist',
|
||||
'❌ Discards a trace that matches the docs line for line, so the docs would also need fixing',
|
||||
'Net: trading trust in the checked-in docs against knowledge only you have about the live SDK.',
|
||||
];
|
||||
const tail = parts.at(-1)!.split('\n').map(line => line.trim());
|
||||
return tail.length === explanatory.length && tail.every((line,i) => line === explanatory[i]);
|
||||
}
|
||||
|
||||
/** A missing release measurement is new work even though its benchmark already exists. */
|
||||
function answeredMeasurementGate(fp: AskUserQuestionFingerprint): boolean {
|
||||
const call = fp.nativeCall;
|
||||
if (!call?.sessionId || !call.toolUseId || call.answered !== true || call.failed !== false ||
|
||||
call.questions.length !== 1 || fp.signature !== `${call.sessionId}:${call.toolUseId}` ||
|
||||
!Array.isArray(call.unansweredQuestionIndices) || call.unansweredQuestionIndices.length ||
|
||||
Object.keys(call.answers ?? {}).length !== 1 || !Number.isFinite(Date.parse(call.answeredAt ?? ''))) return false;
|
||||
const q = call.questions[0]!;
|
||||
if (q.header !== 'Measurement' || q.multiSelect || q.options.length < 2 || q.options.length > 4 ||
|
||||
new Set(q.options.map(o => o.label)).size !== q.options.length || fp.options.length !== q.options.length ||
|
||||
!fp.options.every((o,i) => o.index === i+1 && o.label === q.options[i]!.label) ||
|
||||
!q.options.some(o => call.answers?.[q.question] === o.label) || /<gstack-qid/i.test(q.question)) return false;
|
||||
const title = q.question.split('\n')[0]!.replace(/^D\d+\s*[—–-]\s*/, '');
|
||||
return /^Pass \d+ \(DX Measurement\): the < \d+(?:\.\d+)? min target is asserted but never re-measured after the fixes\.$/.test(title) &&
|
||||
/^Evidence: [^\n]+\. Nothing in the plan re-runs that same study after D\d+[–-]D\d+ land, so the beta could ship with the target still unmet and nobody would know until the survey\.$/m.test(q.question) &&
|
||||
q.options.some(o => /^Fix in plan: re-run study as ship gate, record demo and live TTHW(?: \(recommended\))?$/.test(o.label) &&
|
||||
/^Same protocol as docs\/benchmarks\.md on the release candidate; demo TTHW < \d+(?:\.\d+)? min required before tagging\.$/.test(o.description ?? ''));
|
||||
}
|
||||
|
||||
/** A recap can confirm existing approvals, but its text cannot manufacture them. */
|
||||
function answeredRoleplayRecap(fp: AskUserQuestionFingerprint, priorCalls: readonly NativePlanQuestionCall[]): boolean {
|
||||
const call = fp.nativeCall;
|
||||
const completed = (c: NativePlanQuestionCall) => c.answered === true && c.failed === false &&
|
||||
Boolean(c.sessionId && c.toolUseId) && c.questions.length === 1 && !c.questions[0]!.multiSelect &&
|
||||
Array.isArray(c.unansweredQuestionIndices) && c.unansweredQuestionIndices.length === 0 &&
|
||||
Object.keys(c.answers ?? {}).length === 1 && Number.isFinite(Date.parse(c.answeredAt ?? '')) &&
|
||||
c.questions[0]!.options.filter(o => o.label === c.answers?.[c.questions[0]!.question]).length === 1;
|
||||
if (!call || !completed(call) || fp.signature !== `${call.sessionId}:${call.toolUseId}` ||
|
||||
(fp.nativeQuestionIndex !== undefined && fp.nativeQuestionIndex !== 0)) return false;
|
||||
const q = call.questions[0]!;
|
||||
if (q.header !== 'Roleplay' || q.options.length !== 4 || fp.options.length !== 4 ||
|
||||
!fp.options.every((o, i) => o.index === i + 1 && o.label === q.options[i]!.label) ||
|
||||
new Set(q.options.map(o => o.label)).size !== 4 || /<gstack-qid/i.test(q.question)) return false;
|
||||
const labels = q.options.map(o => o.label.replace(/ \(recommended\)$/i, ''));
|
||||
if (labels.join('|') !== 'All of them, fix every confusion point|Let me pick which ones matter|Critical ones only (#1, #2, #5)|This is unrealistic, our developers already know the context' ||
|
||||
call.answers?.[q.question] !== q.options[0]!.label) return false;
|
||||
const mapping = /^Address #1 through #(\d+), matching the D(\d+)[–-]D(\d+) decisions\.$/.exec(q.options[0]!.description ?? '');
|
||||
if (!mapping) return false;
|
||||
const [size, first, last] = mapping.slice(1).map(Number);
|
||||
if (size !== 5 || last! - first! + 1 !== size || first! < 1 || last! > 1000) return false;
|
||||
if (q.options[1]!.description !== 'Tell me which numbers to keep and which to drop.' ||
|
||||
q.options[2]!.description !== 'Fix quickstart, CI gate, and upgrade; leave signature order and auth error.' ||
|
||||
q.options[3]!.description !== 'Skip the confusion points; keep contracts as drafted.') return false;
|
||||
const prior: NativePlanQuestionCall[] = [];
|
||||
for (let decision = first!; decision <= last!; decision++) {
|
||||
const matches = priorCalls.filter(c => c.sessionId === call.sessionId && completed(c) &&
|
||||
c.toolUseId !== call.toolUseId && Date.parse(c.answeredAt!) < Date.parse(call.answeredAt!) &&
|
||||
new RegExp(`^D${decision}\\s*[—–-]\\s*`).test(c.questions[0]!.question));
|
||||
if (matches.length !== 1 || !/^Fix in plan:/.test(matches[0]!.answers![matches[0]!.questions[0]!.question]!)) return false;
|
||||
prior.push(matches[0]!);
|
||||
}
|
||||
// Each observed confusion point refers to the same already-approved contract.
|
||||
// The fixture's five independent defects remain explicit; new measurement,
|
||||
// documentation or TODO decisions do not enter this confirmation path.
|
||||
const subjects = [/examples\/first_eval\.py/, /\bCI\b/, /\brun_eval\b[\s\S]*\brun_batch\b|\brun_batch\b[\s\S]*\brun_eval\b/, /\bAuthError\b/, /Client\.evaluate\(\)/i];
|
||||
if (prior.some((c, i) => !subjects[i]!.test(c.questions[0]!.question))) return false;
|
||||
// Sharing a subject or a "fix" prefix is not approval of this remedy. Bind
|
||||
// each chosen option and its entire consequence to the contract recapped.
|
||||
const approvedRepairs = [
|
||||
['Fix in plan: demo-first quickstart + resolve first_eval.py', 'README leads with python -m evalkit.demo; ship or remove first_eval.py; add a packaging check for documented paths.'],
|
||||
['Fix in plan: no CI check on mock-transport runs; gate the first live eval instead', 'Demo returns immediately; CI check with existing progress/timeout messaging moves to the first keyed evaluation.'],
|
||||
['Fix in plan: align order + keyword-only + clear TypeError', 'run_batch(dataset, evaluator) matching run_eval; keyword-only enforcement; positional misuse raises a TypeError naming the expected call.'],
|
||||
['Fix in plan: coded, causal AuthError with fix and redaction', 'Error code, key source, cause, console fix URL, redacted key prefix, help link. Matches the existing error pattern.'],
|
||||
['Fix in plan: alias + DeprecationWarning + migration guide + codemod', 'evaluate() delegates to run() with a warning through 2.x betas; changelog and docs/api.md gain a migration section; sed/codemod recipe shipped.'],
|
||||
];
|
||||
if (prior.some((c, i) => {
|
||||
const question = c.questions[0]!;
|
||||
const selected = question.options.find(o => o.label === c.answers![question.question])!;
|
||||
return selected.label.replace(/ \(recommended\)$/i, '') !== approvedRepairs[i]![0] ||
|
||||
selected.description !== approvedRepairs[i]![1];
|
||||
})) return false;
|
||||
const parts = q.question.replace(/^D\d+\s*[—–-]\s*/, '').split(/\n\s*\n/);
|
||||
if (parts.length !== 5 || parts[0] !== 'First-time developer roleplay: which confusion points should the plan address?' ||
|
||||
!/^Project\/branch\/task: [\w/-]+ branch, [\w. -]+ beta polish; roleplayed your ML engineer through the README as written\.$/.test(parts[1]!) ||
|
||||
parts[2] !== 'I roleplayed as your ML engineer attempting the getting started flow. Here is what confused me, with timestamps:') return false;
|
||||
const observed = parts[3]!.split('\n');
|
||||
const source = String.raw`[\w./-]+:\d+(?:-\d+)?`;
|
||||
const observation = [
|
||||
new RegExp(String.raw`^T\+\d+:\d+ +#1 \x60python examples/first_eval\.py\x60 fails: file not in package or archive \(${source}, ${source}\)\. "[^"\n]+"$`),
|
||||
new RegExp(String.raw`^T\+\d+:\d+ +#2 Keyless demo starts a remote CI check on a sample-project binding I never created \(${source}, ${source}\)\. "[^"\n]+"$`),
|
||||
new RegExp(String.raw`^T\+\d+:\d+ +Scores print\. Works, but \d+ min vs the \d+ min target \(${source}\)\. Impression: slow\.$`),
|
||||
new RegExp(String.raw`^T\+\d+:\d+ +#3 run_batch fails inside the evaluator because its argument order is the reverse of run_eval \(${source}\)\. "[^"\n]+"$`),
|
||||
new RegExp(String.raw`^T\+\d+:\d+ +#4 \x60AuthError: request failed\x60 on a wrong-project key; I check network and server status first because nothing says "key" \(${source}\)\.$`),
|
||||
new RegExp(String.raw`^T\+\d+:\d+ +#5 v1 project upgraded: every client\.evaluate\(\) raises AttributeError; changelog has no migration entry \(${source}\)\. Final state: file an issue or pin v1\.$`),
|
||||
];
|
||||
if (observed.length !== observation.length || observed.some((line, i) => !observation[i]!.test(line))) return false;
|
||||
// Consume the complete decision explanation too. Additional work under a
|
||||
// valid heading or in a choice description must remain substantive.
|
||||
const range = `D${first}–D${last}`;
|
||||
const tail = parts[4]!.replace(new RegExp(`D${first}[–-]D${last}`, 'g'), range).split('\n');
|
||||
const expected = [
|
||||
'ELI10: Each numbered point is a place a real first-time user stops and asks a question nobody is there to answer. The plan should remove every one it reasonably can.',
|
||||
'Stakes if we pick wrong: leave one in and that is the step where the developer\'s session ends; each maps to a contract PLAN.md explicitly asked to be reviewed.',
|
||||
`Recommendation: A because all five map one-to-one to the ${range} decisions you already resolved as "fix in plan", so addressing all of them is consistent with those calls.`,
|
||||
'Completeness: A=10/10, B=depends on selection, C=6/10, D=1/10',
|
||||
'A) All of them, fix every confusion point (recommended)',
|
||||
`✅ Consistent with ${range}; every confusion point already has an agreed fix`,
|
||||
'✅ Leaves no known dead end in the first 30 minutes of use',
|
||||
'❌ Full set of fixes touches README, client.py, demo gate, error class, and changelog (human: ~3 days / CC: ~1.5 hours)',
|
||||
'B) Let me pick which ones matter',
|
||||
'✅ Lets you drop a point if you know something the docs do not show',
|
||||
'✅ Keeps the plan focused on what you consider blocking',
|
||||
`❌ Reopens decisions ${range} that were just settled`,
|
||||
'C) The critical ones only (#1, #2, #5), skip #3 and #4',
|
||||
'✅ Covers the broken quickstart, the TTHW blocker, and the upgrade break',
|
||||
'✅ Smaller diff to review',
|
||||
'❌ Ships an inconsistent API and an undiagnosable auth error in a DX polish release',
|
||||
'D) This is unrealistic, our developers already know the context',
|
||||
'✅ Zero work now',
|
||||
'✅ Valid if every beta user is internal and already trained',
|
||||
'❌ README.md:3-5 describes an external ML engineer meeting the SDK fresh, which contradicts this',
|
||||
'Net: trading a known, already-scoped set of fixes against leaving a documented dead end in the first session.',
|
||||
];
|
||||
return tail.length === expected.length && tail.every((line, i) => line.trim() === expected[i]);
|
||||
}
|
||||
|
||||
/** A batched native call remains one decision; the caller owns call-ID deduplication. */
|
||||
export function isDevexReviewIssue(fp: AskUserQuestionFingerprint): boolean {
|
||||
return answeredSetupRepair(fp) || answeredContractRepair(fp) || answeredKeylessDemoRepair(fp) || answeredDocumentationFollowup(fp) || questionRecords(fp, true).some(substantiveIssue);
|
||||
export function isDevexReviewIssue(fp: AskUserQuestionFingerprint, priorCalls: readonly NativePlanQuestionCall[] = []): boolean {
|
||||
if (answeredQuotedAccuracy(fp) || answeredRoleplayRecap(fp, priorCalls)) return false;
|
||||
return answeredMeasurementGate(fp) || answeredSetupRepair(fp) || answeredContractRepair(fp) || answeredKeylessDemoRepair(fp) || answeredDocumentationFollowup(fp) || questionRecords(fp, true).some(substantiveIssue);
|
||||
}
|
||||
|
||||
/** Key acquisition docs and eliminating the demo's key requirement are distinct work. */
|
||||
|
||||
@@ -69,6 +69,39 @@ export function installDisabledPlanReviewFixture(rendered: string, repo: string,
|
||||
return { workflowPath, instructions, generated, stateDir, cliDispatchLog, reviewLogPath, priorRecord, env };
|
||||
}
|
||||
|
||||
/** Check each completion mention; a historical record cannot excuse a later current claim. */
|
||||
function hasUnattributedOutsideCompletion(output: string): boolean {
|
||||
const marker = /\b(?:both reviewers agree|outside_status["']*\s*[:=]\s*["']*completed)\b/gi;
|
||||
const clauses = output.replace(/[*`]/g, '').split(/\r?\n|(?<=[.!?;])\s+|\b(?:but|however|nevertheless|yet)\b[:,]?\s*/i);
|
||||
return clauses.some(clause => [...clause.matchAll(marker)].some(match => {
|
||||
const before = clause.slice(0, match.index).trimEnd();
|
||||
// A quoted phrase is not automatically a denial. Require the local no-claim
|
||||
// statement, so a second positive assertion in the same paragraph still fails.
|
||||
const denied = /\bno\s*["'“”‘’]*\s*$/i.test(before)
|
||||
|| /\b(?:(?:do|did|will|would|can|could)\s+not|cannot|can't|won't)\s+(?:claim|say|state|report)\s*["'“”‘’]*\s*$/i.test(before)
|
||||
|| /\b(?:am|is|are)\s+not\s+(?:claiming|saying|stating|reporting)\s*["'“”‘’]*\s*$/i.test(before);
|
||||
if (denied) return false;
|
||||
// Agreement is a current prose claim unless explicitly denied; an old
|
||||
// log entry only establishes the provenance of its recorded status value.
|
||||
if (/^both reviewers agree$/i.test(match[0])) return true;
|
||||
const record = [...before.matchAll(/\b(?:earlier|prior|historical|old)\s+(?:entry|record)\b/gi)].at(-1);
|
||||
if (!record) return true;
|
||||
// Bind this occurrence to an old record's reported value. A mere mention
|
||||
// of a record, a second status, or a new reporting subject cannot inherit
|
||||
// its historical attribution, even without a sentence boundary.
|
||||
const prefix = before.slice(record.index + record[0].length);
|
||||
const report = /^(?:\s*,?\s*timestamped\b[^,;.!?]{1,160},?)?\s*(?:(?:claiming|shows?|showed|says?|said|records?|recorded|reported)\b|:)\s*/i.exec(prefix);
|
||||
// Only intervening review-log metadata belongs to this reported value.
|
||||
// Arbitrary prose could switch to a new subject without an earlier status.
|
||||
const field = String.raw`["']?(?:status|source|host|outside_provider|phase|timestamp)["']?\s*[:=]\s*["']?[a-z0-9_.:+-]+["']?`;
|
||||
const metadata = new RegExp(String.raw`^(?:${field}\s*(?:,\s*|(?:with|and)\s*))*$`, 'i');
|
||||
const reportsOldValue = report !== null && metadata.test(prefix.slice(report[0].length).trim());
|
||||
const attribution = clause.slice(record.index).replace(/\bbefore\s+this\s+(?:run|session|workflow)\b/gi, 'beforehand');
|
||||
const current = /\b(?:now|currently|current|today|new|updat\w*|append\w*|chang\w*|mark\w*|set|write|wrote)\b|\bthis\s+(?:run|session|workflow)\b/i.test(attribution);
|
||||
return !reportsOldValue || current;
|
||||
}));
|
||||
}
|
||||
|
||||
export function disabledPlanReviewEvidence(result: {
|
||||
exitReason: string; output: string; transcript: any[];
|
||||
}, cliDispatchLog: string, reviewLog = '', priorRecord?: Record<string, unknown>) {
|
||||
@@ -89,7 +122,7 @@ export function disabledPlanReviewEvidence(result: {
|
||||
const disabledAttribution = /outside_status["'`*]*\s*[:=]\s*["'`*]*disabled\b/i.test(result.output)
|
||||
|| result.output.split('\n').some(line => /\b(?:outside(?:\s+(?:voice|review))?|codex(?:\s+review)?)\b/i.test(line)
|
||||
&& /\bdisabled\b/i.test(line) && !/\bnot\s+disabled\b/i.test(line));
|
||||
const falseCompletion = /\b(?:both reviewers agree|outside_status["'`*]*\s*[:=]\s*["'`*]*completed)\b/i.test(result.output);
|
||||
const falseCompletion = hasUnattributedOutsideCompletion(result.output);
|
||||
// Native CLI releases expose the requested subagent as Agent or Task.
|
||||
// Availability never permits dispatch: fallbackCalls rejects both names.
|
||||
const agentAvailable = Array.isArray(init?.tools) && init.tools.some((tool: unknown) => tool === 'Agent' || tool === 'Task');
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import type { AskUserQuestionFingerprint } from './claude-pty-runner';
|
||||
|
||||
/** Count a completed review-navigation choice separately; never choose pending input. */
|
||||
export function isEngCompletionHandoff(fp: AskUserQuestionFingerprint, reviewedPlan: string): boolean {
|
||||
const call = fp.nativeCall;
|
||||
if (!call?.sessionId || !call.toolUseId || call.answered !== true || call.failed !== false ||
|
||||
fp.signature !== `${call.sessionId}:${call.toolUseId}` || call.questions.length !== 1 ||
|
||||
!Array.isArray(call.unansweredQuestionIndices) || call.unansweredQuestionIndices.length ||
|
||||
(fp.nativeQuestionIndex !== undefined && fp.nativeQuestionIndex !== 0) ||
|
||||
Object.keys(call.answers ?? {}).length !== 1 || !Number.isFinite(Date.parse(call.answeredAt ?? ''))) return false;
|
||||
const q = call.questions[0]!;
|
||||
if (q.multiSelect || q.header.trim() !== 'Next steps' || q.options.length !== 2 ||
|
||||
fp.options.length !== 2 || !fp.options.every((o, i) => o.index === i + 1 && o.label === q.options[i]!.label) ||
|
||||
!q.options.some(o => o.label === call.answers?.[q.question]) || /<gstack-qid/i.test(q.question)) return false;
|
||||
const body = q.question.trim().replace(/^D[1-9]\d*\s*[—–:-]\s*/i, '');
|
||||
const lines = body.split('\n').map(s => s.trim()).filter(Boolean);
|
||||
if (lines.length !== 4 ||
|
||||
!/^Next steps: Eng Review is CLEAR\. This is a backend auth refactor with no UI scope, so \/plan-design-review does not apply\. No CEO review exists, but the plan changes no product direction\. What next\?$/.test(lines[0]!) ||
|
||||
!/^Recommendation: [1-9]\d*[A-Z] because the plan is implementation-ready and a CEO review would add little to a pure infrastructure refactor\.$/.test(lines[1]!) ||
|
||||
!/^Note: options differ in kind, not coverage [—–-] no completeness score\.$/.test(lines[2]!) ||
|
||||
!/^Net: start building now versus one more optional review pass on scope\.$/.test(lines[3]!)) return false;
|
||||
const label = (s: string) => s.trim().replace(/^[1-9]\d*[A-Z]\)\s*/, '').replace(/\s*\(recommended\)$/, '');
|
||||
const ready = q.options.find(o => /^Ready to implement [—–-] run \/ship when done$/.test(label(o.label)));
|
||||
const ceo = q.options.find(o => label(o.label) === 'Run /plan-ceo-review first');
|
||||
if (!ready || !ceo) return false;
|
||||
const task = /^Exit plan mode with the reviewed plan; implement T([1-9]\d*)[–-]T([1-9]\d*) \(record T([1-9]\d*) regression fixtures first\)\. ✅ All required reviews complete and logged\. ✅ Tasks JSONL and QA test plan are already written for \/autoplan and \/qa\. ❌ No second-opinion pass since codex reviews are disabled\.$/.exec(ready.description ?? '');
|
||||
// Bind implementation references to the published reviewed task catalog.
|
||||
// A new task or a newly proposed regression step is still substantive work.
|
||||
if (!task || Number(task[1]) !== 1 || Number(task[2]) < Number(task[3]) || Number(task[3]) < Number(task[1])) return false;
|
||||
const tasks = [...reviewedPlan.matchAll(/^- \[ \] \*\*T([1-9]\d*)\b[^\n]+$/gm)];
|
||||
const ids = tasks.map(t => Number(t[1])).sort((a, b) => a - b);
|
||||
if (ids.length !== Number(task[2]) || ids.some((id, i) => id !== i + 1)) return false;
|
||||
const regression = tasks.find(t => t[1] === task[3])?.[0] ?? '';
|
||||
if (!/ [—–] Record regression(?: characterization)? fixtures before\b/i.test(regression)) return false;
|
||||
return ceo.description === 'Optional scope and strategy pass before implementing. ✅ Catches product-level questions the eng review does not ask. ✅ Adds a CEO row to the dashboard. ❌ Little product surface here; likely confirms the current scope.';
|
||||
}
|
||||
@@ -71,11 +71,17 @@ export function recordFilePermission(input: string, file: string, cwd: string, c
|
||||
}
|
||||
|
||||
/** A long diff can crop its path header; the native access choice repeats the directory. */
|
||||
function croppedEditTarget(screen: string): string | undefined {
|
||||
function croppedEditTarget(screen: string, cwd: string): string | undefined {
|
||||
const text = screen.replace(/\r+\n?/g, '\n');
|
||||
// Cropping may begin inside a wrapped added/deleted diff row (five-space gutter).
|
||||
// Still require numbered rows below and the full native footer; never a quoted AUQ.
|
||||
if (!/^(?:\s*\d+\s+[ +\-]?| {5}[+\-])/.test(text) || /[☐□]|^\s*(?:>|`{3}|~{3})/m.test(text)) return undefined;
|
||||
// The heading can be cropped one row earlier, leaving the complete path.
|
||||
// Keep it only when the existing menu independently identifies that target.
|
||||
const header = /^ {0,3}([^\n]+)\n[╌─━]{3,}[ \t]*\n/.exec(text);
|
||||
const headerPath = header?.[1]?.trim();
|
||||
const pathOnly = headerPath && (path.isAbsolute(headerPath) || /^\.\.?[/\\]/.test(headerPath));
|
||||
const diff = pathOnly ? text.slice(header![0].length) : text;
|
||||
if (!/^(?:\s*\d+\s+[ +\-]?| {5}[+\-])/.test(diff) || /[☐□]|^\s*(?:>|`{3}|~{3})/m.test(text)) return undefined;
|
||||
const prompt = [...text.matchAll(/^ {0,3}Do you want to make this edit to ([^\n?\/\\]+)\?[ \t]*\n([\s\S]*)$/gm)].at(-1);
|
||||
if (!prompt || (text.slice(0, prompt.index).match(/^\s*\d+\s+/gm)?.length ?? 0) < 2) return undefined;
|
||||
// The unselected option supplies path identity only. Input remains one-time Yes.
|
||||
@@ -84,7 +90,8 @@ function croppedEditTarget(screen: string): string | undefined {
|
||||
const choices = /^ {0,3}❯[ \t]*1\.[ \t]*Yes[ \t]*\n\s*2\.[ \t]*Yes,\s+and\s+switch\s+to\s+accept\s+edits\s+\(auto-approve\s+file\s+edits\s+and\s+common\s+file\s+commands\)\s+for\s+this\s+session;\s+Yes,\s+and\s+always\s+allow\s+access\s+to\s+([^\r\n]+?)\s+for\s+this\s+session(?:\s*\(shift\+tab\))?\s*\n\s*3\.[ \t]*No(?:hift\+tab\))?[ \t]*\n\s*Esc to cancel [·•] Tab to amend\s*$/.exec(prompt[2]!);
|
||||
const directory = choices?.[1]?.trim();
|
||||
if (!directory || !path.isAbsolute(directory)) return undefined;
|
||||
return path.join(directory, prompt[1]!.trim());
|
||||
const target = path.join(directory, prompt[1]!.trim());
|
||||
return !pathOnly || path.resolve(cwd, headerPath!) === target ? target : undefined;
|
||||
}
|
||||
|
||||
/** Undefined leaves other permissions alone; null keeps this report pane waiting. */
|
||||
@@ -93,7 +100,7 @@ export function currentFilePermissionEpoch(file: string | undefined, expected: s
|
||||
screen: string): FilePermissionEpoch | null | undefined {
|
||||
if (!file || !expected || !config) return undefined;
|
||||
const panel = [...screen.matchAll(/(?:^|\n) {0,3}(?:Edit|Write) file[ \t]*\n {0,3}([^\n]+)\n/g)].at(-1);
|
||||
const target = panel ? path.resolve(cwd,panel[1]!.trim()) : croppedEditTarget(screen);
|
||||
const target = panel ? path.resolve(cwd,panel[1]!.trim()) : croppedEditTarget(screen, cwd);
|
||||
if (target !== expected) {
|
||||
// A foreign path with this report's basename cannot fall back to a stale
|
||||
// owned grant. An incomplete owned menu also waits for full path identity.
|
||||
|
||||
@@ -139,7 +139,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// Each one tests behavior the SDK harness can't observe (rendered TTY,
|
||||
// numbered-option lists, multi-phase ordering, idempotency state echo).
|
||||
'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-ask-user-question-format-compliance.test.ts'],
|
||||
'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/ceo-mode-option.ts', 'test/ceo-mode-option.test.ts', 'test/pty-option-selection.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-ceo-mode-routing.test.ts', 'test/ceo-mode-posture-native.test.ts', 'test/fixtures/ceo-hold-posture-l.json', 'test/ceo-mode-labels-native.test.ts', 'test/fixtures/ceo-mode-labels-l.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/ceo-mode-prerequisite.test.ts', 'test/fixtures/ceo-mode-prerequisite-o-calls.json', 'test/fixtures/ceo-mode-prerequisite-q-call.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/ceo-posture-packet.test.ts', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/ceo-mode-preview-aa-screen.txt', 'test/fixtures/ceo-count-mode-preview-aa-screen.txt', 'test/ceo-expansion-auq.test.ts', 'test/fixtures/ceo-expansion-auq-ac.json', 'test/helpers/plan-count-pending-question.ts', 'test/autoplan-pending-question.test.ts', 'test/plan-pending-question-pty.test.ts', 'test/ceo-barless-submit.test.ts', 'test/fixtures/ceo-barless-submit-ac.json', 'test/pending-question-completion.test.ts', 'test/fixtures/pending-question-completion-ad.json', 'test/ceo-mode-posture-ad.test.ts', 'test/fixtures/ceo-mode-posture-ad.json', 'test/ceo-mode-full-ad.test.ts', 'test/fixtures/ceo-mode-full-ad.json'],
|
||||
'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/ceo-mode-option.ts', 'test/ceo-mode-option.test.ts', 'test/pty-option-selection.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-ceo-mode-routing.test.ts', 'test/ceo-mode-posture-native.test.ts', 'test/fixtures/ceo-hold-posture-l.json', 'test/ceo-mode-labels-native.test.ts', 'test/fixtures/ceo-mode-labels-l.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/ceo-mode-prerequisite.test.ts', 'test/fixtures/ceo-mode-prerequisite-o-calls.json', 'test/fixtures/ceo-mode-prerequisite-q-call.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/ceo-posture-packet.test.ts', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/ceo-mode-preview-aa-screen.txt', 'test/fixtures/ceo-count-mode-preview-aa-screen.txt', 'test/ceo-expansion-auq.test.ts', 'test/fixtures/ceo-expansion-auq-ac.json', 'test/helpers/plan-count-pending-question.ts', 'test/autoplan-pending-question.test.ts', 'test/plan-pending-question-pty.test.ts', 'test/ceo-barless-submit.test.ts', 'test/fixtures/ceo-barless-submit-ac.json', 'test/pending-question-completion.test.ts', 'test/fixtures/pending-question-completion-ad.json', 'test/ceo-mode-posture-ad.test.ts', 'test/fixtures/ceo-mode-posture-ad.json', 'test/ceo-mode-full-ad.test.ts', 'test/fixtures/ceo-mode-full-ad.json', 'test/ceo-prerequisite-ad-v2.test.ts', 'test/fixtures/ceo-prerequisite-ad-v2.json'],
|
||||
'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-with-ui.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'],
|
||||
'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-ship-idempotency.test.ts', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt'],
|
||||
'tpa-present': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts', 'test/helpers/third-party-actions.ts'],
|
||||
@@ -154,16 +154,16 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// the selector sets GSTACK_CARVE_SKILL=<name> to scope cost to the changed
|
||||
// skill (D-CODEX A). Touching the registry/helper or sections.ts runs all.
|
||||
'carve-section-loading': ['scripts/resolvers/composition.ts', 'test/autoplan-review-discovery.test.ts', 'test/autoplan-phase-order.test.ts', 'design-html/**', 'design-shotgun/**', 'qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/resolvers/redact-doc.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/session-runner-tools.test.ts', 'test/carve-section-loading.test.ts', 'bin/gstack-autoplan-snapshot.ts', 'test/autoplan-snapshot.test.ts', 'test/autoplan-init.test.ts', 'test/autoplan-obligations.test.ts', 'test/fixtures/autoplan/t-ceo-omitted-obligations.json', 'test/fixtures/autoplan/u-ceo-original-loss.json', 'test/fixtures/autoplan/v-ceo-dangling-references.json'],
|
||||
'autoplan-chain-pty': ['scripts/resolvers/composition.ts', 'test/autoplan-review-discovery.test.ts', 'test/autoplan-phase-order.test.ts', 'autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/autoplan-dashboard.md', 'test/autoplan-chain-fixture.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'bin/gstack-config', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/autoplan-setup-question.ts', 'test/autoplan-setup-question.test.ts', 'test/helpers/autoplan-phase-observer.ts', 'test/autoplan-phase-observer.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/skill-e2e-autoplan-chain.test.ts', 'bin/gstack-autoplan-snapshot.ts', 'test/autoplan-snapshot.test.ts', 'test/autoplan-init.test.ts', 'test/autoplan-obligations.test.ts', 'test/fixtures/autoplan/t-ceo-omitted-obligations.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/helpers/eval-budgets.ts', 'test/autoplan-eval-budget.test.ts', 'test/eval-budgets-policy.test.ts', 'test/eval-detach-timeout-floor.test.ts', 'scripts/test-paid-shards.ts', '.github/workflows/evals-periodic.yml', 'test/fixtures/autoplan-routing-n-screen.txt', 'test/autoplan-routing-o.test.ts', 'test/fixtures/autoplan-routing-o-screen.txt', 'test/autoplan-setup-packet-o.test.ts', 'test/fixtures/autoplan-setup-packet-o-screen.txt', 'test/fixtures/autoplan-setup-packet-o-call.json', 'test/fixtures/autoplan/u-ceo-original-loss.json', 'test/fixtures/autoplan/v-ceo-dangling-references.json', 'test/fixtures/autoplan-setup-z-packet.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/helpers/autoplan-method-read-audit.ts', 'test/autoplan-method-read-audit.test.ts', 'test/fixtures/autoplan-method-read-aa-events.json', 'test/autoplan-routing-manual-skills.test.ts', 'test/fixtures/autoplan-routing-manual-skills-ac.json', 'test/helpers/plan-count-pending-question.ts', 'test/autoplan-pending-question.test.ts', 'test/plan-pending-question-pty.test.ts', 'test/pending-question-completion.test.ts', 'test/fixtures/pending-question-completion-ad.json'],
|
||||
'autoplan-chain-pty': ['scripts/resolvers/composition.ts', 'test/autoplan-review-discovery.test.ts', 'test/autoplan-phase-order.test.ts', 'autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/autoplan-dashboard.md', 'test/autoplan-chain-fixture.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'bin/gstack-config', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/autoplan-setup-question.ts', 'test/autoplan-setup-question.test.ts', 'test/helpers/autoplan-phase-observer.ts', 'test/autoplan-phase-observer.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/skill-e2e-autoplan-chain.test.ts', 'bin/gstack-autoplan-snapshot.ts', 'test/autoplan-snapshot.test.ts', 'test/autoplan-init.test.ts', 'test/autoplan-obligations.test.ts', 'test/fixtures/autoplan/t-ceo-omitted-obligations.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/helpers/eval-budgets.ts', 'test/autoplan-eval-budget.test.ts', 'test/eval-budgets-policy.test.ts', 'test/eval-detach-timeout-floor.test.ts', 'scripts/test-paid-shards.ts', '.github/workflows/evals-periodic.yml', 'test/fixtures/autoplan-routing-n-screen.txt', 'test/autoplan-routing-o.test.ts', 'test/fixtures/autoplan-routing-o-screen.txt', 'test/autoplan-setup-packet-o.test.ts', 'test/fixtures/autoplan-setup-packet-o-screen.txt', 'test/fixtures/autoplan-setup-packet-o-call.json', 'test/fixtures/autoplan/u-ceo-original-loss.json', 'test/fixtures/autoplan/v-ceo-dangling-references.json', 'test/fixtures/autoplan-setup-z-packet.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/helpers/autoplan-method-read-audit.ts', 'test/autoplan-method-read-audit.test.ts', 'test/fixtures/autoplan-method-read-aa-events.json', 'test/autoplan-routing-manual-skills.test.ts', 'test/fixtures/autoplan-routing-manual-skills-ac.json', 'test/helpers/plan-count-pending-question.ts', 'test/autoplan-pending-question.test.ts', 'test/plan-pending-question-pty.test.ts', 'test/pending-question-completion.test.ts', 'test/fixtures/pending-question-completion-ad.json', 'test/fixtures/autoplan-setup-ad-v2-packet.json'],
|
||||
|
||||
// Per-finding AskUserQuestion count + review-report-at-bottom assertion.
|
||||
// Each test drives its skill end-to-end; touchfiles include preamble +
|
||||
// completion-status resolvers because they affect question cadence and
|
||||
// terminal output (the regression surface this test catches).
|
||||
'plan-ceo-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'test/skill-ceo-section-ordering.test.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/ceo-completion-handoff.ts', 'test/ceo-completion-handoff.test.ts', 'test/ceo-count-s-terminals.test.ts', 'test/fixtures/ceo-count-s-paired.json', 'test/fixtures/ceo-completion-handoff-calls.json', 'test/fixtures/ceo-completion-handoff-j-calls.json', 'test/fixtures/ceo-completion-handoff-k-calls.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-ceo-finding-count.test.ts', 'test/ceo-completion-handoff-l.test.ts', 'test/fixtures/ceo-completion-handoff-l-calls.json', 'test/ceo-completion-handoff-m.test.ts', 'test/fixtures/ceo-completion-handoff-m-call.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/ceo-handoff-n-calls.json', 'test/ceo-completion-handoff-o.test.ts', 'test/fixtures/ceo-completion-handoff-o-call.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/helpers/ceo-approach-pick.ts', 'test/ceo-approach-pick.test.ts', 'test/fixtures/ceo-approach-q-call.json', 'test/fixtures/ceo-approach-r-call.json', 'test/fixtures/ceo-approach-r-distinct-call.json', 'test/fixtures/ceo-approach-q-paired-call.json', 'test/fixtures/ceo-completion-handoff-q-call.json', 'test/fixtures/ceo-completion-handoff-r-calls.json', 'test/fixtures/ceo-completion-handoff-t-call.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/ceo-completion-handoff-u-call.json', 'test/fixtures/ceo-completion-handoff-v-call.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/plan-count-ceo-body-finding.test.ts', 'test/fixtures/ceo-count-w-paired.json', 'test/fixtures/ceo-completion-handoff-w-call.json', 'test/fixtures/ceo-approach-y-call.json', 'test/fixtures/ceo-approach-y-screen.txt', 'test/ceo-handoff-y.test.ts', 'test/fixtures/ceo-handoff-y-call.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/ceo-handoff-z-call.json', 'test/fixtures/ceo-approach-aa-call.json', 'test/review-handoffs-aa.test.ts', 'test/fixtures/review-handoff-aa-ceo.json', 'test/fixtures/review-handoff-aa-dx.json', 'test/helpers/ceo-mode-option.ts', 'test/ceo-count-mode.test.ts', 'test/fixtures/ceo-count-mode-ab-call.json', 'test/ceo-count-ac.test.ts', 'test/fixtures/ceo-count-ac-calls.json', 'test/fixtures/ceo-count-ac-later-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json'],
|
||||
'plan-eng-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-eng-finding-count.test.ts', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/eng-first-review-t.test.ts', 'test/fixtures/eng-batching-t-calls.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/eng-scope-y.test.ts', 'test/fixtures/eng-scope-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/eng-binding-z.test.ts', 'test/fixtures/eng-binding-z-calls.json', 'test/eng-binding-retry-z.test.ts', 'test/fixtures/eng-binding-retry-z-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json'],
|
||||
'plan-design-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/design-count-review.ts', 'test/design-count-review.test.ts', 'test/fixtures/design-review-j-calls.json', 'test/helpers/design-count-outside.ts', 'test/design-count-outside.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts', 'test/fixtures/design-review-l-calls.json', 'test/design-completion-handoff.test.ts', 'test/fixtures/design-handoff-l-calls.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/design-review-n-calls.json', 'test/design-completion-handoff-scored.test.ts', 'test/fixtures/design-handoff-n-calls.json', 'test/fixtures/design-handoff-q-calls.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/design-completion-handoff-u.test.ts', 'test/fixtures/design-handoff-u-calls.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/helpers/design-artifact-question.ts', 'test/design-artifact-question.test.ts', 'test/fixtures/design-artifacts-w-calls.json', 'test/fixtures/design-outside-y-calls.json', 'test/fixtures/design-boundaries-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/design-gap-z-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json'],
|
||||
'plan-devex-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/devex-count-fixture.ts', 'test/devex-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts', 'test/fixtures/dx-prerequisite-r-call.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/devex-review-n-calls.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/devex-output-o.test.ts', 'test/fixtures/devex-review-o-calls.json', 'test/fixtures/devex-output-o-retry-call.json', 'test/devex-setup-remedy-o.test.ts', 'test/fixtures/devex-review-o-retry-calls.json', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/fixtures/devex-review-t-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/devex-count-u-calls.json', 'test/fixtures/devex-count-u-retry-calls.json', 'test/fixtures/devex-empathy-v-calls.json', 'test/fixtures/devex-handoff-v-call.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/fixtures/devex-count-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/devex-count-z-calls.json', 'test/fixtures/devex-handoff-z-call.json', 'test/review-handoffs-aa.test.ts', 'test/fixtures/review-handoff-aa-ceo.json', 'test/fixtures/review-handoff-aa-dx.json', 'test/devex-empathy-ab.test.ts', 'test/fixtures/devex-empathy-ab-calls.json', 'test/devex-ac-accounting.test.ts', 'test/fixtures/devex-ac-first-attempt-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json'],
|
||||
'plan-ceo-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'test/skill-ceo-section-ordering.test.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/ceo-completion-handoff.ts', 'test/ceo-completion-handoff.test.ts', 'test/ceo-count-s-terminals.test.ts', 'test/fixtures/ceo-count-s-paired.json', 'test/fixtures/ceo-completion-handoff-calls.json', 'test/fixtures/ceo-completion-handoff-j-calls.json', 'test/fixtures/ceo-completion-handoff-k-calls.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-ceo-finding-count.test.ts', 'test/ceo-completion-handoff-l.test.ts', 'test/fixtures/ceo-completion-handoff-l-calls.json', 'test/ceo-completion-handoff-m.test.ts', 'test/fixtures/ceo-completion-handoff-m-call.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/ceo-handoff-n-calls.json', 'test/ceo-completion-handoff-o.test.ts', 'test/fixtures/ceo-completion-handoff-o-call.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/helpers/ceo-approach-pick.ts', 'test/ceo-approach-pick.test.ts', 'test/fixtures/ceo-approach-q-call.json', 'test/fixtures/ceo-approach-r-call.json', 'test/fixtures/ceo-approach-r-distinct-call.json', 'test/fixtures/ceo-approach-q-paired-call.json', 'test/fixtures/ceo-completion-handoff-q-call.json', 'test/fixtures/ceo-completion-handoff-r-calls.json', 'test/fixtures/ceo-completion-handoff-t-call.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/ceo-completion-handoff-u-call.json', 'test/fixtures/ceo-completion-handoff-v-call.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/plan-count-ceo-body-finding.test.ts', 'test/fixtures/ceo-count-w-paired.json', 'test/fixtures/ceo-completion-handoff-w-call.json', 'test/fixtures/ceo-approach-y-call.json', 'test/fixtures/ceo-approach-y-screen.txt', 'test/ceo-handoff-y.test.ts', 'test/fixtures/ceo-handoff-y-call.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/ceo-handoff-z-call.json', 'test/fixtures/ceo-approach-aa-call.json', 'test/review-handoffs-aa.test.ts', 'test/fixtures/review-handoff-aa-ceo.json', 'test/fixtures/review-handoff-aa-dx.json', 'test/helpers/ceo-mode-option.ts', 'test/ceo-count-mode.test.ts', 'test/fixtures/ceo-count-mode-ab-call.json', 'test/ceo-count-ac.test.ts', 'test/fixtures/ceo-count-ac-calls.json', 'test/fixtures/ceo-count-ac-later-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/ceo-count-ad-v2.test.ts', 'test/fixtures/ceo-count-ad-v2.json', 'test/fixtures/plan-count-permission-target-ad-v2.json'],
|
||||
'plan-eng-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-eng-finding-count.test.ts', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/eng-first-review-t.test.ts', 'test/fixtures/eng-batching-t-calls.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/eng-scope-y.test.ts', 'test/fixtures/eng-scope-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/eng-binding-z.test.ts', 'test/fixtures/eng-binding-z-calls.json', 'test/eng-binding-retry-z.test.ts', 'test/fixtures/eng-binding-retry-z-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/fixtures/plan-count-permission-target-ad-v2.json', 'test/helpers/eng-completion-handoff.ts', 'test/eng-count-ad-v2.test.ts', 'test/fixtures/eng-count-ad-v2.json'],
|
||||
'plan-design-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/design-count-review.ts', 'test/design-count-review.test.ts', 'test/fixtures/design-review-j-calls.json', 'test/helpers/design-count-outside.ts', 'test/design-count-outside.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts', 'test/fixtures/design-review-l-calls.json', 'test/design-completion-handoff.test.ts', 'test/fixtures/design-handoff-l-calls.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/design-review-n-calls.json', 'test/design-completion-handoff-scored.test.ts', 'test/fixtures/design-handoff-n-calls.json', 'test/fixtures/design-handoff-q-calls.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/design-completion-handoff-u.test.ts', 'test/fixtures/design-handoff-u-calls.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/helpers/design-artifact-question.ts', 'test/design-artifact-question.test.ts', 'test/fixtures/design-artifacts-w-calls.json', 'test/fixtures/design-outside-y-calls.json', 'test/fixtures/design-boundaries-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/design-gap-z-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/fixtures/plan-count-permission-target-ad-v2.json', 'test/design-count-ad-v2.test.ts', 'test/fixtures/design-count-ad-v2.json'],
|
||||
'plan-devex-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/devex-count-fixture.ts', 'test/devex-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts', 'test/fixtures/dx-prerequisite-r-call.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/devex-review-n-calls.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/devex-output-o.test.ts', 'test/fixtures/devex-review-o-calls.json', 'test/fixtures/devex-output-o-retry-call.json', 'test/devex-setup-remedy-o.test.ts', 'test/fixtures/devex-review-o-retry-calls.json', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/fixtures/devex-review-t-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/devex-count-u-calls.json', 'test/fixtures/devex-count-u-retry-calls.json', 'test/fixtures/devex-empathy-v-calls.json', 'test/fixtures/devex-handoff-v-call.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/fixtures/devex-count-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/devex-count-z-calls.json', 'test/fixtures/devex-handoff-z-call.json', 'test/review-handoffs-aa.test.ts', 'test/fixtures/review-handoff-aa-ceo.json', 'test/fixtures/review-handoff-aa-dx.json', 'test/devex-empathy-ab.test.ts', 'test/fixtures/devex-empathy-ab-calls.json', 'test/devex-ac-accounting.test.ts', 'test/fixtures/devex-ac-first-attempt-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/fixtures/plan-count-permission-target-ad-v2.json', 'test/devex-reconfirmation-ad-v2.test.ts', 'test/fixtures/devex-reconfirmation-ad-v2.json', 'test/plan-count-history.test.ts'],
|
||||
|
||||
// Gate-tier reviewCount-floor counterparts. Catch the May 2026 transcript
|
||||
// bug (model wrote a plan-mode plan and ExitPlanMode'd without firing any
|
||||
@@ -180,8 +180,8 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// a model fires one AUQ then batches the rest into a "## Decisions to
|
||||
// confirm" plan write. runPlanSkillFloorCheck cannot detect that shape
|
||||
// (it exits on first AUQ); runPlanSkillCounting can.
|
||||
'plan-eng-multi-finding-batching': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/eng-first-review-t.test.ts', 'test/fixtures/eng-batching-t-calls.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/eng-scope-y.test.ts', 'test/fixtures/eng-scope-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/eng-binding-z.test.ts', 'test/fixtures/eng-binding-z-calls.json', 'test/eng-binding-retry-z.test.ts', 'test/fixtures/eng-binding-retry-z-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json'],
|
||||
'plan-ceo-split-overflow': ['bin/gstack-config', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json'],
|
||||
'plan-eng-multi-finding-batching': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/eng-first-review-t.test.ts', 'test/fixtures/eng-batching-t-calls.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/eng-scope-y.test.ts', 'test/fixtures/eng-scope-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/eng-binding-z.test.ts', 'test/fixtures/eng-binding-z-calls.json', 'test/eng-binding-retry-z.test.ts', 'test/fixtures/eng-binding-retry-z-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/fixtures/plan-count-permission-target-ad-v2.json', 'test/eng-count-ad-v2.test.ts', 'test/fixtures/eng-count-ad-v2.json'],
|
||||
'plan-ceo-split-overflow': ['bin/gstack-config', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/fixtures/plan-count-permission-target-ad-v2.json'],
|
||||
'brain-privacy-gate': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-brain-privacy-gate.test.ts'],
|
||||
|
||||
// /setup-gbrain Path 4 (Remote MCP) — happy + bad-token end-to-end via
|
||||
@@ -301,7 +301,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'bin/gstack-config', 'bin/gstack-codex-probe', 'bin/gstack-review-log', 'bin/gstack-slug', 'bin/gstack-wtree', 'bin/gstack-brain-enqueue', 'test/helpers/session-runner.ts',
|
||||
'test/helpers/hermetic-env.ts', 'test/helpers/skill-fixture.ts', 'test/helpers/outside-voice-evidence.ts',
|
||||
'test/helpers/disabled-plan-review-fixture.ts', 'test/disabled-plan-review-evidence.test.ts',
|
||||
'test/skill-e2e-outside-plan-disabled.test.ts',
|
||||
'test/skill-e2e-outside-plan-disabled.test.ts', 'test/fixtures/disabled-plan-attribution-ad-v2.json',
|
||||
],
|
||||
|
||||
// GPT-5.6 Sol scope-termination E2E (Codex CLI, full generated investigate skill)
|
||||
|
||||
@@ -67,11 +67,12 @@ describe('native repeated report permission identity',()=>{
|
||||
});
|
||||
});
|
||||
|
||||
for (const variant of ['basic', 'intervening', 'cropped', 'same-basename']) test.skipIf(process.platform==='win32')(`real fake CLI grants each current request once: ${variant}`,async()=>{
|
||||
for (const variant of ['basic', 'intervening', 'cropped', 'same-basename', 'path-cropped']) test.skipIf(process.platform==='win32')(`real fake CLI grants each current request once: ${variant}`,async()=>{
|
||||
const intervening = variant === 'intervening';
|
||||
const dir=fs.mkdtempSync(path.join(os.tmpdir(),'count-edit-pty-'));const fake=path.join(dir,'fake-claude');const worker=path.join(dir,'worker.ts');const events=path.join(dir,'events.jsonl');const output=path.join(dir,'output.json');const expected=path.join(dir,variant==='same-basename'?'PLAN.md':'report.md');fs.writeFileSync(expected,'original');
|
||||
const cropped=capturedAc.rows.find(row=>row.job===5)!;
|
||||
let screen=variant==='cropped' ? cropped.screen.replaceAll(path.dirname(cropped.hook.expected),path.dirname(expected)).replaceAll(path.basename(cropped.hook.expected),'report.md')
|
||||
let screen=variant==='path-cropped' ? capturedPath.screen.replace(capturedPath.screen.split('\n')[0]!,expected).replaceAll(path.dirname(capturedPath.expected),path.dirname(expected)).replaceAll(path.basename(capturedPath.expected),'report.md')
|
||||
: variant==='cropped' ? cropped.screen.replaceAll(path.dirname(cropped.hook.expected),path.dirname(expected)).replaceAll(path.basename(cropped.hook.expected),'report.md')
|
||||
: captured.screen.replaceAll(captured.expectedPath,expected).replace('../gstack-e2e-plan-ceo-paired-2Rv5Bi/gstack-test-plan-ceo-paired.md',expected).replaceAll('gstack-test-plan-ceo-paired.md','report.md');
|
||||
if(variant==='same-basename')screen=screen.replaceAll(expected,'__ACTIVE_PLAN_PATH__').replaceAll('report.md','PLAN.md');
|
||||
fs.writeFileSync(fake,`#!${process.execPath}\n`+String.raw`
|
||||
@@ -111,3 +112,70 @@ process.stdin.setRawMode?.(true);process.stdin.on('data',async data=>{
|
||||
expect(()=>process.kill(rows[0].pid,0)).toThrow();expect(fs.existsSync(rows[0].cwd)).toBe(false);
|
||||
}finally{clearTimeout(killer);child.kill('SIGKILL');if(fs.existsSync(events)){const first=JSON.parse(fs.readFileSync(events,'utf8').split('\n')[0]!);try{process.kill(first.pid,'SIGKILL');}catch{}}fs.rmSync(dir,{recursive:true,force:true});}
|
||||
},35000);
|
||||
|
||||
import capturedPath from './fixtures/plan-count-permission-target-ad-v2.json';
|
||||
function pathCroppedFixture(){
|
||||
const dir=fs.mkdtempSync(path.join(os.tmpdir(),'count-path-cropped-'));
|
||||
// Retain the captured relative layout inside this run's disposable temp root.
|
||||
const relocate=(value:string)=>value.replaceAll(path.dirname(capturedPath.cwd),dir);
|
||||
const cwd=relocate(capturedPath.cwd),config=relocate(capturedPath.config),expected=relocate(capturedPath.expected);
|
||||
const screen=relocate(capturedPath.screen),sessionId=capturedPath.sessionId;
|
||||
const recorder=createFilePermissionRecorder(cwd,config,expected)!;const startedAt=Date.now()-1000;
|
||||
const transcript:any={status:'ready',calls:[],assistantMessages:[{sessionId,text:'Reviewing',timestamp:new Date().toISOString()}]};
|
||||
// The public screen/identity are retained; these hook events are synthetic.
|
||||
const record=(name:string,id:string)=>recordFilePermission(JSON.stringify({hook_event_name:name,tool_name:'Edit',session_id:sessionId,tool_use_id:id,cwd,transcript_path:path.join(config,'projects','owned',sessionId+'.jsonl'),tool_input:{file_path:expected}}),recorder.file,cwd,config,expected);
|
||||
const read=(viewport=screen)=>currentFilePermissionEpoch(recorder.file,expected,cwd,config,startedAt,transcript,viewport);
|
||||
return {recorder,startedAt,transcript,record,read,screen,expected,close(){recorder.dispose();fs.rmSync(dir,{recursive:true,force:true});}};
|
||||
}
|
||||
test('AD v2 retained path-only heading needs the exact current owned epoch',()=>{
|
||||
const f=pathCroppedFixture();try{
|
||||
expect(capturedPath.provenance.actualOutcome).toBe('timeout');
|
||||
expect(classifyPlanCountFrame(f.screen)).toBe('permission');
|
||||
expect(f.read()).toBeNull();f.record('PreToolUse','first');
|
||||
expect(f.read()?.pendingId).toBe(capturedPath.sessionId+':first');
|
||||
const guard=createPlanCountPermissionGuard();expect(guard(f.screen,'',f.read())).toBe('grant');
|
||||
expect(guard(f.screen,'',f.read())).toBe('handled');
|
||||
}finally{f.close();}
|
||||
});
|
||||
|
||||
test('AD v2 path-only target retains native success and replay boundaries',()=>{
|
||||
const f=pathCroppedFixture();try{
|
||||
const guard=createPlanCountPermissionGuard();const read=()=>guard(f.screen,'',f.read());
|
||||
f.record('PreToolUse','first');expect(read()).toBe('grant');
|
||||
f.record('PostToolUseFailure','first');f.record('PreToolUse','second');expect(read()).toBe('handled');
|
||||
f.record('PostToolUse','second');f.record('PreToolUse','third');expect(read()).toBe('handled');
|
||||
}finally{f.close();}
|
||||
const g=pathCroppedFixture();try{
|
||||
const guard=createPlanCountPermissionGuard();g.record('PreToolUse','first');expect(guard(g.screen,'',g.read())).toBe('grant');
|
||||
g.record('PostToolUse','first');expect(guard(g.screen,'',g.read())).toBe('handled');
|
||||
g.record('PreToolUse','second');expect(guard(g.screen,'',g.read())).toBe('grant');
|
||||
g.record('PostToolUse','second');g.record('PreToolUse','first');expect(guard(g.screen,'',g.read())).toBe('handled');
|
||||
}finally{g.close();}
|
||||
});
|
||||
|
||||
test('AD v2 path-only heading must agree with full menu target and current metadata',()=>{
|
||||
const f=pathCroppedFixture();try{
|
||||
f.record('PreToolUse','first');const screen=f.screen,lines=screen.split('\n');
|
||||
expect(f.read([f.expected,...lines.slice(1)].join('\n'))?.pendingId).toBe(capturedPath.sessionId+':first');
|
||||
const wrong=[
|
||||
[' ../sibling/'+path.basename(f.expected),...lines.slice(1)].join('\n'),
|
||||
[' '+f.expected+' extra',...lines.slice(1)].join('\n'),
|
||||
[lines[0],'/tmp/other/'+path.basename(f.expected),...lines.slice(1)].join('\n'),
|
||||
[lines[0],...lines.slice(2)].join('\n'),
|
||||
'Example:\n'+screen,'```text\n'+screen,screen+'\nContinue with a different action.',
|
||||
screen.replace('❯ 1. Yes','❯ 2. Yes'),screen.replace('Esc to cancel · Tab to amend','Esc to cancel'),
|
||||
screen.replace('always allow access to '+path.dirname(f.expected),'always allow access to /tmp/sibling'),
|
||||
];
|
||||
for(const altered of wrong){expect(altered).not.toBe(screen);expect(f.read(altered)).toBeNull();}
|
||||
const valid=JSON.parse(fs.readFileSync(f.recorder.file,'utf8'));
|
||||
for(const delta of [{timestamp:new Date(f.startedAt-1).toISOString()},{sessionId:'foreign'},{pendingId:valid.completedId},{cwd:'/foreign'},{expected:'/foreign'}]){
|
||||
fs.writeFileSync(f.recorder.file,JSON.stringify({...valid,...delta}));expect(f.read()).toBeNull();
|
||||
}
|
||||
}finally{f.close();}
|
||||
});
|
||||
|
||||
import {E2E_TOUCHFILES,selectTests} from './helpers/touchfiles';
|
||||
test('AD v2 cropped target fixture selects all existing file-permission consumers',()=>{
|
||||
expect(selectTests(['test/fixtures/plan-count-permission-target-ad-v2.json'],E2E_TOUCHFILES,[]).selected)
|
||||
.toEqual(selectTests(['test/plan-count-file-permission.test.ts'],E2E_TOUCHFILES,[]).selected);
|
||||
});
|
||||
@@ -0,0 +1,50 @@
|
||||
/** Exercise preceding-call context through the real PTY/native capture loop, without a provider. */
|
||||
import {expect,test} from 'bun:test';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import {pathToFileURL} from 'node:url';
|
||||
|
||||
test.skipIf(process.platform === 'win32')('native issue callbacks receive earlier calls, never the current or future call', async()=>{
|
||||
const dir=fs.mkdtempSync(path.join(os.tmpdir(),'plan-count-history-'));
|
||||
const fake=path.join(dir,'fake-claude'), worker=path.join(dir,'worker.ts');
|
||||
const report=path.join(dir,'report.md'), resultFile=path.join(dir,'result.json'), seen=path.join(dir,'history.jsonl');
|
||||
const runner=pathToFileURL(path.join(import.meta.dir,'helpers/claude-pty-runner.ts')).href;
|
||||
fs.writeFileSync(fake, `#!${process.execPath}\n`+String.raw`
|
||||
import fs from 'node:fs'; import path from 'node:path';
|
||||
const sessionId='history-fixture';
|
||||
const dir=path.join(process.env.CLAUDE_CONFIG_DIR,'projects',sessionId);
|
||||
fs.mkdirSync(dir,{recursive:true}); const file=path.join(dir,sessionId+'.jsonl');
|
||||
const native=(role,content,timestamp,extra={})=>fs.appendFileSync(file,JSON.stringify({
|
||||
cwd:process.cwd(),sessionId,isSidechain:false,timestamp,message:{role,content},...extra,
|
||||
})+'\n');
|
||||
let sent=false;process.stdin.setRawMode?.(true);
|
||||
process.stdin.on('data',()=>{
|
||||
if(sent)return;sent=true;const now=Date.now();
|
||||
for(const [i,id] of ['first','second'].entries()){
|
||||
const q={header:'Finding',question:'Fix issue '+id+'?',options:[{label:'Fix it'},{label:'Keep it'}],multiSelect:false};
|
||||
native('assistant',[{type:'tool_use',id,name:'AskUserQuestion',input:{questions:[q]}}],new Date(now-1000+i*100).toISOString());
|
||||
native('user',[{type:'tool_result',tool_use_id:id,content:'Answered.'}],new Date(now-950+i*100).toISOString(),{toolUseResult:{answers:{[q.question]:q.options[0].label}}});
|
||||
}
|
||||
fs.writeFileSync(process.env.PROBE_PLAN,'# Reviewed plan\n\n## GSTACK REVIEW REPORT\n\n| Review | Status | Findings |\n|---|---|---|\n| DX Review | clean | two resolved |\n\nVERDICT: DX CLEARED — eng review required\n\nNO UNRESOLVED DECISIONS\n');
|
||||
const text='DX review complete. Two findings resolved interactively.\n\nPlan written to: '+String.fromCharCode(96)+process.env.PROBE_PLAN+String.fromCharCode(96);
|
||||
native('assistant',[{type:'text',text}],new Date(Date.now()+5).toISOString());
|
||||
process.stdout.write('●'+text.replace(/ /g,'')+'\nCrunched ·done\n❯ ');
|
||||
});process.stdin.resume();
|
||||
`);
|
||||
fs.chmodSync(fake,0o755);
|
||||
fs.writeFileSync(worker,`import fs from 'node:fs';\nimport {runPlanSkillCounting} from ${JSON.stringify(runner)};\n`+
|
||||
`const result=await runPlanSkillCounting({skillName:'plan-devex-review',slashCommand:'/plan-devex-review',followUpPrompt:'# History fixture',expectedPlanPath:${JSON.stringify(report)},isLastStep0AUQ:()=>false,isReviewAUQ:(fp,prior)=>{fs.appendFileSync(${JSON.stringify(seen)},JSON.stringify({current:fp.nativeCall.toolUseId,prior:prior?.map(c=>c.toolUseId)??null})+'\\n');return true;},reviewCountCeiling:8,timeoutMs:33000,env:{PROBE_PLAN:${JSON.stringify(report)}}});\n`+
|
||||
`fs.writeFileSync(${JSON.stringify(resultFile)},JSON.stringify(result));\n`);
|
||||
const child=Bun.spawn([process.execPath,worker],{env:{...process.env,EVALS_HERMETIC:'1',EVALS_RUN_ID:'',BROWSE_TERMINAL_BINARY:fake},stdout:'pipe',stderr:'pipe'});
|
||||
const timer=setTimeout(()=>child.kill('SIGKILL'),35000);
|
||||
try{
|
||||
const [code,stdout,stderr]=await Promise.all([child.exited,new Response(child.stdout).text(),new Response(child.stderr).text()]);
|
||||
expect(code,stdout+stderr).toBe(0);
|
||||
const obs=JSON.parse(fs.readFileSync(resultFile,'utf8'));
|
||||
expect(obs.outcome).toBe('completion_summary');expect(obs.reviewCount).toBe(2);
|
||||
expect(fs.readFileSync(seen,'utf8').trim().split('\n').map(line=>JSON.parse(line))).toEqual([
|
||||
{current:'first',prior:[]},{current:'second',prior:['first']},
|
||||
]);
|
||||
} finally {clearTimeout(timer);child.kill('SIGKILL');fs.rmSync(dir,{recursive:true,force:true});}
|
||||
},40000);
|
||||
@@ -97,6 +97,7 @@ const planDesign5Findings = (planPath: string) => [
|
||||
// The skill explicitly supports this opt-out. The cadence test measures
|
||||
// review findings; generated-variant preference questions have separate evals.
|
||||
'Text-only review; skip mockups. Review all seven design dimensions.',
|
||||
'Native design review only: skip optional outside voices. Finish after the design review; I will handle subsequent reviews manually.',
|
||||
'',
|
||||
'# Plan: Settings Page UI redesign',
|
||||
'',
|
||||
|
||||
@@ -21,6 +21,8 @@ import {
|
||||
assertReviewReportAtBottom,
|
||||
} from './helpers/claude-pty-runner';
|
||||
|
||||
import { isEngCompletionHandoff } from './helpers/eng-completion-handoff';
|
||||
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
const N = 5;
|
||||
@@ -88,6 +90,10 @@ describeE2E('/plan-eng-review per-finding AskUserQuestion count (periodic)', ()
|
||||
isLastStep0AUQ: engStep0Boundary,
|
||||
isSetupAUQ: engSetupAUQ,
|
||||
isFirstReviewAUQ: engFirstReviewAUQ,
|
||||
isCompletionHandoffAUQ: fp => {
|
||||
try { return isEngCompletionHandoff(fp, fs.readFileSync(planPath, 'utf8')); }
|
||||
catch { return false; } // Unpublished work cannot establish a closed handoff.
|
||||
},
|
||||
reviewCountCeiling: CEILING + 1,
|
||||
timeoutMs: 1_500_000,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
|
||||
Reference in new issue
Block a user