mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-06 08:54:04 +01:00
test: fix native review setup, completion, and isolation failures
This commit is contained in:
Garry Tan committed
1 parent
860f385a82
commit
c54f24934a
32 files changed
+3563
-76
No files matched your search
@@ -0,0 +1,159 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { autoplanSetupDecision } from './helpers/autoplan-setup-question';
|
||||
import { E2E_TOUCHFILES } from './helpers/touchfiles';
|
||||
|
||||
const frame = fs.readFileSync(path.join(import.meta.dir, 'fixtures/autoplan-routing-o-screen.txt'), 'utf8');
|
||||
const question = {
|
||||
header: 'Routing rules',
|
||||
question: "gstack works best when your project's CLAUDE.md includes skill routing rules. Add them now?",
|
||||
options: [{ label: 'Add routing rules (Recommended)' }, { label: 'Skip for now' }],
|
||||
};
|
||||
const native = () => ({ sessionId: 'o-routing', toolUseId: 'routing', answered: false, failed: false, questions: [structuredClone(question)] });
|
||||
|
||||
describe('complete routing panel with a temporary decline', () => {
|
||||
test('the exact O panel chooses Add once before native persistence and after matching persistence', () => {
|
||||
expect(frame).toContain('Invoke skills manually going forward.');
|
||||
for (const pending of [undefined, native()]) {
|
||||
const seen = new Set<string>();
|
||||
const decision = autoplanSetupDecision(frame, seen, pending);
|
||||
expect(decision).toMatchObject({ kind: 'input', input: '1' });
|
||||
expect(seen.size).toBe(0);
|
||||
if (decision.kind !== 'input') throw Error('Expected input');
|
||||
for (const signature of decision.signatures) seen.add(signature);
|
||||
expect(autoplanSetupDecision(frame, seen, pending).kind).toBe('waiting');
|
||||
expect(autoplanSetupDecision(frame, seen, native()).kind).toBe('waiting');
|
||||
}
|
||||
});
|
||||
|
||||
test('the unambiguous opposed decline does not depend on its description or choice order', () => {
|
||||
const withoutDescription = frame.replace(/^\s+Invoke skills manually going forward\..*$/m, '');
|
||||
for (const label of ['Skip for now', 'Skip for now (Recommended)', 'SKIP FOR NOW']) {
|
||||
const current = withoutDescription.replace('2. Skip for now', '2. ' + label);
|
||||
expect(autoplanSetupDecision(current, new Set())).toMatchObject({ kind: 'input', input: '1' });
|
||||
const reversed = current.replace('1. Add routing rules (Recommended)', '1. ' + label)
|
||||
.replace('2. ' + label, '2. Add routing rules (Recommended)');
|
||||
expect(autoplanSetupDecision(reversed, new Set())).toMatchObject({ kind: 'input', input: '2' });
|
||||
}
|
||||
for (const label of ['Skip', 'No thanks', 'Skip — invoke skills manually', 'Manual only']) {
|
||||
expect(autoplanSetupDecision(frame.replace('Skip for now', label), new Set())).toMatchObject({ kind: 'input', input: '1' });
|
||||
}
|
||||
});
|
||||
|
||||
test('extra actions, unrelated questions and ambiguous offered choices do not acquire input', () => {
|
||||
for (const label of ['Skip for now and delete CLAUDE.md', 'Skip for now, implement the feature', 'Skip the review for now', 'Skip for now unless the API changes', 'Ask me after this review']) {
|
||||
expect(autoplanSetupDecision(frame.replace('2. Skip for now', '2. ' + label), new Set()).kind, label).not.toBe('input');
|
||||
}
|
||||
for (const changed of [
|
||||
frame.replace(question.question, 'Which product API routing design should we choose?'),
|
||||
frame.replace(question.question, 'The plan quotes gstack skill routing rules in CLAUDE.md. Should we build an API router?'),
|
||||
frame.replace('1. Add routing rules (Recommended)', '1. Implement routing (Recommended)'),
|
||||
frame.replace('2. Skip for now', '2. Add routing rules'),
|
||||
frame.replace('3. Type something.', '3. Skip for now\n 4. Type something.').replace('4. Chat about this', '5. Chat about this'),
|
||||
frame.replace('3. Type something.', '3. Implement the feature\n 4. Type something.').replace('4. Chat about this', '5. Chat about this'),
|
||||
]) expect(autoplanSetupDecision(changed, new Set()).kind, changed).not.toBe('input');
|
||||
});
|
||||
|
||||
test('only the complete current native panel can supply this additional label', () => {
|
||||
const panel = frame.slice(frame.indexOf(' ☐ Routing rules'));
|
||||
for (const changed of [
|
||||
'Example panel:\n' + panel, 'Quoted source:\n' + panel, '```text\n' + panel, '~~~~text\n' + panel,
|
||||
panel.split('\n').map(line => ' ' + line).join('\n'), panel.split('\n').map(line => '> ' + line).join('\n'),
|
||||
panel + '\n● Continuing the review.', panel.replace('Esc to cancel', 'Esc to'),
|
||||
panel.replace(' 4. Chat about this', ''), panel.replace(' 3. Type something.', ''),
|
||||
panel.replace('❯ 1.', ' 1.'), panel.replace(' 2.', '❯ 2.'),
|
||||
panel.replace('1. Add', '1. [ ] Add'), panel.replace(' ☐ Routing rules', '← ☐ Routing rules ✔ Submit →'),
|
||||
]) expect(autoplanSetupDecision(changed, new Set()).kind, changed).not.toBe('input');
|
||||
expect(autoplanSetupDecision('```text\nold code\n```\n' + panel, new Set())).toMatchObject({kind:'input',input:'1'});
|
||||
});
|
||||
|
||||
test('present metadata cannot be replaced by the visible decline label', () => {
|
||||
for (const mutate of [
|
||||
(call:any) => {call.failed=true;}, (call:any) => {call.answered=true;},
|
||||
(call:any) => {call.questions=[];}, (call:any) => {call.questions.push(structuredClone(question));},
|
||||
(call:any) => {call.questions[0].multiSelect=true;}, (call:any) => {call.questions[0].header='Other';},
|
||||
(call:any) => {call.questions[0].question='Different question';},
|
||||
(call:any) => {call.questions[0].options[1].label='Different choice';},
|
||||
]) {const call=native();mutate(call);expect(autoplanSetupDecision(frame,new Set(),call).kind).not.toBe('input');}
|
||||
});
|
||||
});
|
||||
|
||||
test('routing regression inputs remain paid-selection dependencies', () => {
|
||||
expect(E2E_TOUCHFILES['autoplan-chain-pty']).toContain('test/autoplan-routing-o.test.ts');
|
||||
expect(E2E_TOUCHFILES['autoplan-chain-pty']).toContain('test/fixtures/autoplan-routing-o-screen.txt');
|
||||
});
|
||||
|
||||
test.skipIf(process.platform === 'win32')('real PTY temporary routing decline advances after readiness with exactly one Add digit', async () => {
|
||||
const dir=fs.mkdtempSync(path.join(os.tmpdir(),'gstack-routing-o-'));
|
||||
const fake=path.join(dir,'fake-claude');const worker=path.join(dir,'worker.ts');const output=path.join(dir,'result.json');
|
||||
const cases=[false,true].map(early=>({name:early?'early':'deferred',early,frame,question,
|
||||
cwd:path.join(dir,early?'early':'deferred'),events:path.join(dir,early?'early.jsonl':'deferred.jsonl')}));
|
||||
for(const item of cases)fs.mkdirSync(item.cwd);
|
||||
fs.writeFileSync(fake,`#!${process.execPath}\n`+String.raw`
|
||||
import * as fs from 'node:fs';import * as path from 'node:path';
|
||||
const item=JSON.parse(process.env.ROUTING_CASE);const event=value=>fs.appendFileSync(item.events,JSON.stringify(value)+'\n');
|
||||
const folder=path.join(process.env.CLAUDE_CONFIG_DIR,'projects','fixture');fs.mkdirSync(folder,{recursive:true});
|
||||
const file=path.join(folder,item.name+'.jsonl');
|
||||
const persist=value=>fs.appendFileSync(file,JSON.stringify({sessionId:item.name,isSidechain:false,cwd:process.cwd(),timestamp:new Date().toISOString(),...value})+'\n');
|
||||
const use=()=>persist({type:'assistant',message:{role:'assistant',content:[{type:'tool_use',id:'routing',name:'AskUserQuestion',input:{questions:[item.question]}}]}});
|
||||
event({kind:'startup',pid:process.pid});if(item.early)use();
|
||||
process.stdin.setRawMode?.(true);process.stdin.resume();
|
||||
process.stdin.on('data',data=>{event({kind:'input',data:data.toString()});if(!item.early)use();
|
||||
persist({type:'user',toolUseResult:{answers:{[item.question.question]:'Add routing rules (Recommended)'}},message:{role:'user',content:[{type:'tool_result',tool_use_id:'routing',content:'User has answered your questions: "'+item.question.question+'"="Add routing rules (Recommended)". You can now continue with the user\'s answers in mind.'}]}});
|
||||
process.stdout.write('\r\nROUTING_ACCEPTED\r\n');});
|
||||
process.stdout.write('\x1b[2J\x1b[H'+item.frame.replace(/\n/g,'\r\n'));
|
||||
process.on('SIGINT',()=>process.exit(0));
|
||||
`);fs.chmodSync(fake,0o755);
|
||||
const url=(name:string)=>pathToFileURL(path.resolve(import.meta.dir,'helpers',name)).href;
|
||||
fs.writeFileSync(worker,`
|
||||
import * as fs from 'node:fs';
|
||||
import {launchClaudePty,resolveClaudeBinary} from ${JSON.stringify(url('claude-pty-runner.ts'))};
|
||||
import {readPlanCountTranscript} from ${JSON.stringify(url('plan-count-transcript.ts'))};
|
||||
import {autoplanSetupDecision} from ${JSON.stringify(url('autoplan-setup-question.ts'))};
|
||||
if(resolveClaudeBinary()!==${JSON.stringify(fake)})throw Error('Fake binary binding failed before launch');
|
||||
const results=[];
|
||||
for(const item of ${JSON.stringify(cases)}){
|
||||
const session=await launchClaudePty({cwd:item.cwd,observeScreen:true,timeoutMs:15000,env:{ROUTING_CASE:JSON.stringify(item)}});
|
||||
try{
|
||||
await session.waitFor('Enter to select',{timeoutMs:10000,pollMs:20});
|
||||
const screen=await session.currentScreen();const before=readPlanCountTranscript(session.hermeticConfigDir,item.cwd);
|
||||
const pending=before.calls.find(call=>!call.answered&&!call.failed);
|
||||
if(Boolean(pending)!==item.early)throw Error('Incorrect readiness metadata');
|
||||
const seen=new Set();const decision=autoplanSetupDecision(screen,seen,pending);
|
||||
if(decision.kind!=='input'||decision.input!=='1')throw Error('Expected Add input: '+JSON.stringify(decision));
|
||||
session.send(decision.input);for(const signature of decision.signatures)seen.add(signature);
|
||||
await session.waitFor('ROUTING_ACCEPTED',{timeoutMs:3000,pollMs:20});
|
||||
const after=readPlanCountTranscript(session.hermeticConfigDir,item.cwd);
|
||||
results.push({name:item.name,decision,after,redraw:autoplanSetupDecision(screen,seen,pending).kind,
|
||||
answered:autoplanSetupDecision(screen,new Set(),after.calls[0]).kind});
|
||||
}finally{await session.close();}
|
||||
}
|
||||
fs.writeFileSync(${JSON.stringify(output)},JSON.stringify(results));
|
||||
`);
|
||||
const child=Bun.spawn([process.execPath,worker],{env:{...process.env,BROWSE_TERMINAL_BINARY:fake},stdout:'pipe',stderr:'pipe'});
|
||||
const killer=setTimeout(()=>child.kill('SIGKILL'),25000);
|
||||
try{
|
||||
const [exit,stdout,stderr]=await Promise.all([child.exited,new Response(child.stdout).text(),new Response(child.stderr).text()]);
|
||||
expect(exit,stdout+stderr).toBe(0);
|
||||
const results=JSON.parse(fs.readFileSync(output,'utf8'));
|
||||
expect(results.length).toBe(2);
|
||||
for(const [index,result]of results.entries()){
|
||||
expect(result.decision).toMatchObject({kind:'input',input:'1'});expect(result.redraw).toBe('waiting');expect(result.answered).toBe('waiting');
|
||||
expect(result.after.calls.length).toBe(1);expect(result.after.calls[0].answered).toBe(true);
|
||||
expect(result.after.calls[0].answers[question.question]).toBe('Add routing rules (Recommended)');
|
||||
const events=fs.readFileSync(cases[index]!.events,'utf8').trim().split('\n').map(line=>JSON.parse(line));
|
||||
expect(events.filter(event=>event.kind==='input')).toEqual([{kind:'input',data:'1'}]);
|
||||
expect(()=>process.kill(events[0].pid,0)).toThrow();
|
||||
}
|
||||
}finally{
|
||||
clearTimeout(killer);child.kill('SIGKILL');
|
||||
for(const item of cases)if(fs.existsSync(item.events)){
|
||||
const pid=JSON.parse(fs.readFileSync(item.events,'utf8').split('\n')[0]!).pid;
|
||||
if(process.platform==='linux')try{if(fs.readFileSync('/proc/'+pid+'/cmdline','utf8').split('\0').includes(fake))process.kill(pid,'SIGKILL');}catch{}
|
||||
}
|
||||
fs.rmSync(dir,{recursive:true,force:true});
|
||||
}
|
||||
},30000);
|
||||
@@ -0,0 +1,193 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { autoplanSetupDecision } from './helpers/autoplan-setup-question';
|
||||
import { E2E_TOUCHFILES } from './helpers/touchfiles';
|
||||
import type { NativePlanQuestionCall } from './helpers/plan-count-transcript';
|
||||
|
||||
const captured = fs.readFileSync(path.join(import.meta.dir, 'fixtures/autoplan-setup-packet-o-screen.txt'), 'utf8');
|
||||
const original = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/autoplan-setup-packet-o-call.json'), 'utf8')) as NativePlanQuestionCall;
|
||||
const footer = 'Enter to select · Tab/Arrow keys to navigate · Esc to cancel';
|
||||
function pane(call: NativePlanQuestionCall, index: number, answered: number[] = []) {
|
||||
const bar = '← ' + call.questions.map((q,i) => (answered.includes(i) ? '☒ ' : '☐ ') + q.header).join(' ') + ' ✔ Submit →';
|
||||
if (index === call.questions.length) return `${bar}\nReview your answers\nReady to submit your answers?\n❯ 1. Submit answers\n 2. Cancel\n${footer}\n`;
|
||||
const q = call.questions[index]!;
|
||||
return `${bar}\n│ ${q.question}\n` + q.options.map((option,i) => `${i===0?'❯':' '} ${i+1}. ${option.label}`).join('\n') +
|
||||
`\n 3. Type something.\n 4. Chat about this\n${footer}\n`;
|
||||
}
|
||||
function commit(screen: string, seen: Set<string>, call: NativePlanQuestionCall, expected: string) {
|
||||
const before = [...seen]; const action = autoplanSetupDecision(screen, seen, call);
|
||||
expect([...seen]).toEqual(before); expect(action).toMatchObject({kind:'input',input:expected});
|
||||
if (action.kind !== 'input') throw Error('Expected input');
|
||||
for (const key of action.signatures) seen.add(key);
|
||||
expect(autoplanSetupDecision(screen,seen,call).kind).toBe('waiting');
|
||||
return action;
|
||||
}
|
||||
|
||||
describe('native routing and prerequisite setup packet', () => {
|
||||
test('exact O active pane then prerequisite each receive one bound choice, followed by one Submit', () => {
|
||||
const seen=new Set<string>();
|
||||
commit(captured,seen,original,'1');
|
||||
expect(autoplanSetupDecision(pane(original,2,[0,1]),seen,original).kind).toBe('waiting');
|
||||
commit(pane(original,1,[0]),seen,original,'1');
|
||||
commit(pane(original,2,[0,1]),seen,original,'\r');
|
||||
expect(autoplanSetupDecision(captured,new Set(),{...original,answered:true}).kind).toBe('waiting');
|
||||
});
|
||||
|
||||
test('question and option order may change without changing the existing choices', () => {
|
||||
for (const reverseQuestions of [false,true]) for (const reverseOptions of [false,true]) {
|
||||
const call=structuredClone(original);if(reverseQuestions)call.questions.reverse();
|
||||
if(reverseOptions)for(const question of call.questions)question.options.reverse();
|
||||
const seen=new Set<string>();
|
||||
for(let index=0;index<2;index++)commit(pane(call,index,index?[0]:[]),seen,call,reverseOptions?'2':'1');
|
||||
commit(pane(call,2,[0,1]),seen,call,'\r');
|
||||
}
|
||||
});
|
||||
|
||||
test('metadata may persist late, but no tab is answered before the complete packet is known', () => {
|
||||
const seen=new Set<string>();
|
||||
expect(autoplanSetupDecision(captured,seen).kind).toBe('waiting');expect(seen.size).toBe(0);
|
||||
expect(autoplanSetupDecision(pane(original,0).split('\n').slice(1).join('\n'),seen).kind).toBe('waiting');
|
||||
expect(autoplanSetupDecision(captured,seen,{...original,questions:[original.questions[0]!]}).kind).toBe('waiting');
|
||||
commit(captured,seen,original,'1');
|
||||
expect(autoplanSetupDecision(pane(original,2,[0,1]),new Set(),original).kind).toBe('waiting');
|
||||
});
|
||||
|
||||
test('every native question must be one unambiguous setup offer', () => {
|
||||
for (const mutate of [
|
||||
(call:any)=>{call.failed=true;},(call:any)=>{call.answered=true;},(call:any)=>{call.questions[1].multiSelect=true;},
|
||||
(call:any)=>{call.questions.push(structuredClone(call.questions[0]));},
|
||||
(call:any)=>{call.questions[1]=structuredClone(call.questions[0]);},
|
||||
(call:any)=>{call.questions[1].question='Which user experience should the API provide?';},
|
||||
(call:any)=>{call.questions[1].question='No design doc exists for /office-hours integration. Should we build X or defer Y?';},
|
||||
(call:any)=>{call.questions[0].question+=' Should we delete the archived invoices?';},
|
||||
(call:any)=>{call.questions[0].question+=' Also approve deleting the archived invoices before continuing.';},
|
||||
(call:any)=>{call.questions[1].question='Should we delete the archived invoices? '+call.questions[1].question;},
|
||||
(call:any)=>{call.questions[1].question=call.questions[1].question.replace('— sharper input','and also approve deleting the archived invoices — sharper input');},
|
||||
(call:any)=>{call.questions[1].question='No design doc found for this branch. /office-hours produces a design doc — also archive the invoices. Run it first or proceed with standard review?';},
|
||||
(call:any)=>{call.questions[1].options[1].label='Run /office-hours first then implement';},
|
||||
(call:any)=>{call.questions[1].options[0].label='Skip — implement the feature';},
|
||||
(call:any)=>{call.questions[0].question='The plan quotes gstack skill routing rules in CLAUDE.md. Should we build an API router?';},
|
||||
(call:any)=>{call.questions[0].options[1].label='No thanks, delete CLAUDE.md';},
|
||||
(call:any)=>{call.questions[0].options.push({label:'Implement the feature'});},
|
||||
]) {const call=structuredClone(original);mutate(call);const seen=new Set<string>();
|
||||
expect(autoplanSetupDecision(pane(call,0),seen,call).kind).toBe('waiting');expect(seen.size).toBe(0);}
|
||||
});
|
||||
|
||||
test('current tab, full offered labels and active panel context must all agree', () => {
|
||||
const first=pane(original,0);
|
||||
for (const changed of [
|
||||
'Example panel:\n'+first, 'Example:\n'+first, 'Quoted source:\n'+first, '```text\n'+first, '~~~~text\n'+first,
|
||||
first.split('\n').map(line=>' '+line).join('\n'), first.split('\n').map(line=>'> '+line).join('\n'),
|
||||
first+'\n● Continuing the review.', first+first, first.replace('Esc to cancel','Esc to'),
|
||||
first.replace('Prerequisite doc','Other tab'), first.replace(original.questions[0]!.question,'Unrelated question'),
|
||||
first.replace('← ', '← Different call '),
|
||||
first.replace('1. Add','1. Delete'), first.replace('1. Add','1. [ ] Add'),
|
||||
first.replace(' 3. Type something.',''), first.replace(' 4. Chat about this',''),
|
||||
first.replace('❯ 1.',' 1.'),first.replace(' 2.','❯ 2.'),
|
||||
first.replace(' 3. Type something.',' 3. Implement the feature\n 4. Type something.').replace(' 4. Chat about this',' 5. Chat about this'),
|
||||
]) expect(autoplanSetupDecision(changed,new Set(),original).kind,changed).toBe('waiting');
|
||||
expect(autoplanSetupDecision('```text\nearlier code\n```\n'+first,new Set(),original)).toMatchObject({kind:'input',input:'1'});
|
||||
expect(autoplanSetupDecision(pane(original,0,[0]),new Set(),original).kind).toBe('waiting');
|
||||
});
|
||||
|
||||
test('Submit requires each actual sent identity, checked tabs, unchanged packet and a current Submit panel', () => {
|
||||
const seen=new Set<string>();commit(pane(original,0),seen,original,'1');commit(pane(original,1,[0]),seen,original,'1');
|
||||
const submit=pane(original,2,[0,1]);
|
||||
for (const changed of [
|
||||
pane(original,2,[0]), 'Example panel:\n'+submit,'Example:\n'+submit,'```text\n'+submit,submit+'\n● Finished.',
|
||||
submit.replace('Submit answers','Accept implementation'),submit.replace('Ready to submit your answers?','Implement the feature?'),
|
||||
submit.replace(' 2. Cancel',' 2. Cancel\n 3. Deploy'),submit.replace('Esc to cancel','Esc to'),
|
||||
]) expect(autoplanSetupDecision(changed,seen,original).kind,changed).toBe('waiting');
|
||||
for (const change of ['session','tool','question','description']) {
|
||||
const call=structuredClone(original);
|
||||
if(change==='session')call.sessionId+='-other';if(change==='tool')call.toolUseId+='-other';
|
||||
if(change==='question')call.questions[0]!.question+=' ';
|
||||
if(change==='description')call.questions[0]!.options[0]!.description='Changed';
|
||||
expect(autoplanSetupDecision(submit,seen,call).kind).toBe('waiting');
|
||||
}
|
||||
commit(submit,seen,original,'\r');
|
||||
});
|
||||
|
||||
test('packet captures and regression remain paid-selection dependencies', () => {
|
||||
for(const file of ['test/autoplan-setup-packet-o.test.ts','test/fixtures/autoplan-setup-packet-o-screen.txt','test/fixtures/autoplan-setup-packet-o-call.json'])
|
||||
expect(E2E_TOUCHFILES['autoplan-chain-pty']).toContain(file);
|
||||
});
|
||||
});
|
||||
|
||||
test.skipIf(process.platform==='win32')('real PTY native setup packet waits for metadata, answers each visible tab once and submits without a stray digit',async()=>{
|
||||
const dir=fs.mkdtempSync(path.join(os.tmpdir(),'gstack-setup-packet-o-'));const fake=path.join(dir,'fake-claude');
|
||||
const worker=path.join(dir,'worker.ts');const resultFile=path.join(dir,'results.json');
|
||||
const cases=[false,true].map(late=>({name:late?'late':'early',late,cwd:path.join(dir,late?'late':'early'),
|
||||
events:path.join(dir,late?'late.jsonl':'early.jsonl'),release:path.join(dir,late?'late.release':'early.release'),call:original,first:captured}));
|
||||
for(const item of cases)fs.mkdirSync(item.cwd);
|
||||
fs.writeFileSync(fake,`#!${process.execPath}\n`+String.raw`
|
||||
import * as fs from 'node:fs';import * as path from 'node:path';
|
||||
const item=JSON.parse(process.env.PACKET_CASE);const event=value=>fs.appendFileSync(item.events,JSON.stringify(value)+'\n');
|
||||
const folder=path.join(process.env.CLAUDE_CONFIG_DIR,'projects','fixture');fs.mkdirSync(folder,{recursive:true});
|
||||
const file=path.join(folder,item.call.sessionId+'.jsonl');let index=0,answers={},published=false,done=false;
|
||||
const persist=value=>fs.appendFileSync(file,JSON.stringify({sessionId:item.call.sessionId,isSidechain:false,cwd:process.cwd(),timestamp:new Date().toISOString(),...value})+'\n');
|
||||
const publish=()=>{if(published)return;published=true;persist({type:'assistant',message:{role:'assistant',content:[{type:'tool_use',id:item.call.toolUseId,name:'AskUserQuestion',input:{questions:item.call.questions}}]}});event({kind:'metadata'});process.stdout.write('\r\nMETADATA_READY\r\n');render();};
|
||||
function render(){const q=item.call.questions;let screen=item.first;
|
||||
if(index>0){const bar='← '+q.map(question=>(answers[question.question]?'☒ ':'☐ ')+question.header).join(' ')+' ✔ Submit →';
|
||||
screen=index<q.length?bar+'\n│ '+q[index].question+'\n'+q[index].options.map((o,i)=>(i===0?'❯':' ')+' '+(i+1)+'. '+o.label).join('\n')+'\n 3. Type something.\n 4. Chat about this':bar+'\nReview your answers\nReady to submit your answers?\n❯ 1. Submit answers\n 2. Cancel';
|
||||
screen+='\nEnter to select · Tab/Arrow keys to navigate · Esc to cancel\n';}
|
||||
process.stdout.write('\x1b[2J\x1b[H'+screen.replace(/\n/g,'\r\n'));}
|
||||
event({kind:'startup',pid:process.pid});process.stdin.setRawMode?.(true);process.stdin.resume();
|
||||
process.stdin.on('data',data=>{const input=data.toString();event({kind:'input',input,index,published});if(!published||done)throw Error('Unexpected input lifecycle');
|
||||
if(index<2){if(!/^[12]$/.test(input))throw Error('One native digit required');answers[item.call.questions[index].question]=item.call.questions[index].options[Number(input)-1].label;index++;render();}
|
||||
else{if(input!=='\r')throw Error('Raw Submit required');done=true;persist({type:'user',toolUseResult:{answers},message:{role:'user',content:[{type:'tool_result',tool_use_id:item.call.toolUseId,content:'Answered.'}]}});event({kind:'submitted',answers});process.stdout.write('\x1b[2J\x1b[HNATIVE_PACKET_COMPLETE\r\n');}});
|
||||
render();if(!item.late)publish();const timer=setInterval(()=>{if(item.late&&fs.existsSync(item.release))publish();},10);
|
||||
process.on('SIGINT',()=>{clearInterval(timer);process.exit(0);});
|
||||
`);fs.chmodSync(fake,0o755);
|
||||
const url=(name:string)=>pathToFileURL(path.resolve(import.meta.dir,'helpers',name)).href;
|
||||
fs.writeFileSync(worker,`
|
||||
import * as fs from 'node:fs';
|
||||
import {launchClaudePty,resolveClaudeBinary} from ${JSON.stringify(url('claude-pty-runner.ts'))};
|
||||
import {autoplanSetupDecision} from ${JSON.stringify(url('autoplan-setup-question.ts'))};
|
||||
import {readPlanCountTranscript} from ${JSON.stringify(url('plan-count-transcript.ts'))};
|
||||
if(resolveClaudeBinary()!==${JSON.stringify(fake)})throw Error('Fake binary binding failed before launch');
|
||||
const results=[];
|
||||
for(const item of ${JSON.stringify(cases)}){
|
||||
const session=await launchClaudePty({cwd:item.cwd,observeScreen:true,timeoutMs:15000,env:{PACKET_CASE:JSON.stringify(item)}});
|
||||
try{
|
||||
await session.waitFor('Enter to select',{timeoutMs:10000,pollMs:20});const seen=new Set();
|
||||
if(item.late){const pending=readPlanCountTranscript(session.hermeticConfigDir,item.cwd).calls[0];if(pending)throw Error('Expected missing native packet');
|
||||
if(autoplanSetupDecision(await session.currentScreen(),seen,pending).kind!=='waiting'||seen.size)throw Error('Guessed before native identity');
|
||||
fs.writeFileSync(item.release,'release');}
|
||||
await session.waitFor('METADATA_READY',{timeoutMs:3000,pollMs:20});
|
||||
const inputs=[];
|
||||
for(let step=0;step<3;step++){
|
||||
const current=await session.currentScreen();const call=readPlanCountTranscript(session.hermeticConfigDir,item.cwd).calls[0];
|
||||
const action=autoplanSetupDecision(current,seen,call);
|
||||
if(action.kind!=='input')throw Error('Expected input at '+step+': '+JSON.stringify({action,current,call}));
|
||||
session.send(action.input);inputs.push(action.input);for(const signature of action.signatures)seen.add(signature);
|
||||
if(autoplanSetupDecision(current,seen,call).kind!=='waiting')throw Error('Repeated input on unchanged pane');
|
||||
await session.waitFor(step===0?'☒ Routing setup':step===1?'Ready to submit your answers?':'NATIVE_PACKET_COMPLETE',{timeoutMs:3000,pollMs:20});
|
||||
}
|
||||
const transcript=readPlanCountTranscript(session.hermeticConfigDir,item.cwd);results.push({name:item.name,inputs,transcript});
|
||||
}finally{await session.close();}}
|
||||
fs.writeFileSync(${JSON.stringify(resultFile)},JSON.stringify(results));
|
||||
`);
|
||||
const child=Bun.spawn([process.execPath,worker],{env:{...process.env,BROWSE_TERMINAL_BINARY:fake},stdout:'pipe',stderr:'pipe'});
|
||||
const killer=setTimeout(()=>child.kill('SIGKILL'),26000);
|
||||
try{
|
||||
const [exit,stdout,stderr]=await Promise.all([child.exited,new Response(child.stdout).text(),new Response(child.stderr).text()]);expect(exit,stdout+stderr).toBe(0);
|
||||
const results=JSON.parse(fs.readFileSync(resultFile,'utf8'));expect(results.length).toBe(2);
|
||||
for(const [index,result]of results.entries()){
|
||||
expect(result.inputs).toEqual(['1','1','\r']);expect(result.transcript.calls.length).toBe(1);expect(result.transcript.calls[0].answered).toBe(true);
|
||||
expect(result.transcript.calls[0].answers).toEqual(Object.fromEntries(original.questions.map(q=>[q.question,q.options[0]!.label])));
|
||||
const events=fs.readFileSync(cases[index]!.events,'utf8').trim().split('\n').map(line=>JSON.parse(line));
|
||||
expect(events.filter(e=>e.kind==='input').map(e=>({input:e.input,index:e.index,published:e.published}))).toEqual([
|
||||
{input:'1',index:0,published:true},{input:'1',index:1,published:true},{input:'\r',index:2,published:true}]);
|
||||
expect(events.filter(e=>e.kind==='submitted').length).toBe(1);expect(()=>process.kill(events[0].pid,0)).toThrow();
|
||||
}
|
||||
}finally{
|
||||
clearTimeout(killer);child.kill('SIGKILL');for(const item of cases)if(fs.existsSync(item.events)){
|
||||
const pid=JSON.parse(fs.readFileSync(item.events,'utf8').split('\n')[0]!).pid;
|
||||
if(process.platform==='linux')try{if(fs.readFileSync('/proc/'+pid+'/cmdline','utf8').split('\0').includes(fake))process.kill(pid,'SIGKILL');}catch{}
|
||||
}fs.rmSync(dir,{recursive:true,force:true});
|
||||
}
|
||||
},30000);
|
||||
@@ -0,0 +1,152 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { ceoFirstReviewAUQ, ceoStep0Boundary, hasNativePlanTerminal, nativePlanCallFingerprint, planCountQuestionPhase } from './helpers/claude-pty-runner';
|
||||
import { isCeoCompletionHandoff, pickCeoCompletionHandoff } from './helpers/ceo-completion-handoff';
|
||||
import type { NativePlanQuestionCall } from './helpers/plan-count-transcript';
|
||||
import captured from './fixtures/ceo-completion-handoff-o-call.json';
|
||||
|
||||
const calls = () => structuredClone(captured.calls) as NativePlanQuestionCall[];
|
||||
const handoff = () => calls().at(-1)!;
|
||||
const fingerprint = (call: NativePlanQuestionCall) => nativePlanCallFingerprint(call, 0, false);
|
||||
const reanswer = (call: NativePlanQuestionCall) => {
|
||||
const question = call.questions[0]!;
|
||||
call.answers = { [question.question]: question.options[0]!.label };
|
||||
return call;
|
||||
};
|
||||
|
||||
describe('closed CEO navigation with the native review-prefixed identity', () => {
|
||||
test('the actual six-call sequence preserves setup and both independent findings', () => {
|
||||
const original = calls();
|
||||
let started = false;
|
||||
const counts = { setup: 0, review: 0, administrative: 0 };
|
||||
for (const call of original) {
|
||||
const phase = planCountQuestionPhase(fingerprint(call), started, ceoStep0Boundary,
|
||||
ceoFirstReviewAUQ, undefined, isCeoCompletionHandoff);
|
||||
started = phase.reviewStarted;
|
||||
if (phase.administrative) counts.administrative++;
|
||||
else if (phase.preReview) counts.setup++;
|
||||
else counts.review++;
|
||||
}
|
||||
expect(counts).toEqual({ setup: 3, review: 2, administrative: 1 });
|
||||
expect(original).toEqual(calls());
|
||||
expect(original.slice(3, 5).map(call => isCeoCompletionHandoff(fingerprint(call)))).toEqual([false, false]);
|
||||
});
|
||||
|
||||
test('the offered manual action needs the matching pending native call in either order', () => {
|
||||
for (const reverse of [false, true]) {
|
||||
const call = handoff();
|
||||
call.answered = false;
|
||||
delete call.answers;
|
||||
delete call.unansweredQuestionIndices;
|
||||
if (reverse) call.questions[0]!.options.reverse();
|
||||
expect(pickCeoCompletionHandoff(fingerprint(call))).toBe(reverse ? 1 : 2);
|
||||
expect(isCeoCompletionHandoff(fingerprint(call))).toBe(false);
|
||||
expect(pickCeoCompletionHandoff({ ...fingerprint(call), signature: 'foreign:call' })).toBeNull();
|
||||
}
|
||||
expect(pickCeoCompletionHandoff(fingerprint(handoff()))).toBeNull();
|
||||
});
|
||||
|
||||
test('closed navigation semantics are shared across the bounded review identity family', () => {
|
||||
for (const id of ['ceo-review-next-step', 'ceo-review-next-steps', 'ceo-review-next-review', 'ceo-plan-next-steps']) {
|
||||
for (const completion of ['done', 'complete', 'cleared']) {
|
||||
const call = handoff();
|
||||
call.questions[0]!.question = call.questions[0]!.question
|
||||
.replace('ceo-review-next-steps', id).replace('CEO review done.', `CEO review ${completion}.`);
|
||||
expect(isCeoCompletionHandoff(fingerprint(reanswer(call)))).toBe(true);
|
||||
}
|
||||
}
|
||||
const sequencing = handoff();
|
||||
sequencing.questions[0]!.options[1]!.description = 'Once implementation is finished, run /plan-eng-review. After Eng review is complete, proceed to shipping.';
|
||||
expect(isCeoCompletionHandoff(fingerprint(sequencing))).toBe(true);
|
||||
});
|
||||
|
||||
test('a completed heading cannot conceal unresolved work or a substantive question', () => {
|
||||
for (const text of [
|
||||
'CEO review is not done. What\'s next?',
|
||||
'CEO review done only after fixing the missing authorization test. What\'s next?',
|
||||
'CEO review done. Should we add a missing authorization test before Eng?',
|
||||
'CEO review done. We should fix the missing authorization test. What\'s next?',
|
||||
'CEO review done. Do you want me to fix the missing authorization test? What\'s next?',
|
||||
'CEO review done. One contrast issue remains. What\'s next?',
|
||||
'CEO review done. Validation is still pending. What\'s next?',
|
||||
'CEO review done. Not all findings are resolved. What\'s next?',
|
||||
'CEO review done. One test issue is still open. What\'s next?',
|
||||
'CEO review done. There are not 0 unresolved decisions. What\'s next?',
|
||||
'CEO review done. If the tests pass, what\'s next?',
|
||||
'CEO review done. What\'s next? Once the tests pass, all decisions are resolved.',
|
||||
'CEO review done. What\'s next? After the authorization tests pass, the review is complete.',
|
||||
'CEO review done. What\'s next? The review is complete when authorization tests pass.',
|
||||
'CEO review done. What\'s next? Once the tests pass, all decisions will be resolved.',
|
||||
'CEO review done. What\'s next? All findings become resolved after the tests pass.',
|
||||
'Example: CEO review done. What\'s next?',
|
||||
]) {
|
||||
const call = handoff();
|
||||
call.questions[0]!.question = call.questions[0]!.question.replace("CEO review done. What's next?", text);
|
||||
expect(isCeoCompletionHandoff(fingerprint(reanswer(call))), text).toBe(false);
|
||||
call.answered = false;
|
||||
expect(pickCeoCompletionHandoff(fingerprint(call)), text).toBeNull();
|
||||
}
|
||||
for (const description of [
|
||||
'Proceed to fix the missing authorization test before Eng.',
|
||||
'The contrast gap remains unresolved; handle it manually.',
|
||||
'Please add a new regression test before implementation.',
|
||||
'We could add a missing regression test before Eng.',
|
||||
'Do you want to add a new test before the next review?',
|
||||
]) {
|
||||
const call = handoff();
|
||||
call.questions[0]!.options[1]!.description = description;
|
||||
expect(isCeoCompletionHandoff(fingerprint(call)), description).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('failed, partial, malformed, unrelated or mixed native calls remain substantive', () => {
|
||||
for (const mutate of [
|
||||
(call: NativePlanQuestionCall) => { call.failed = true; },
|
||||
(call: NativePlanQuestionCall) => { call.unansweredQuestionIndices = [0]; },
|
||||
(call: NativePlanQuestionCall) => { call.questions[0]!.multiSelect = true; },
|
||||
(call: NativePlanQuestionCall) => { call.questions.push(calls()[3]!.questions[0]!); },
|
||||
(call: NativePlanQuestionCall) => { call.questions[0]!.header = 'Test gap'; },
|
||||
(call: NativePlanQuestionCall) => { call.questions[0]!.question = call.questions[0]!.question.replace('ceo-review-next-steps', 'ceo-review-test-gap'); },
|
||||
(call: NativePlanQuestionCall) => { call.questions[0]!.question += ' <gstack-qid'; },
|
||||
(call: NativePlanQuestionCall) => { call.questions[0]!.question = call.questions[0]!.question.replace('required shipping gate', 'optional review'); },
|
||||
(call: NativePlanQuestionCall) => { call.questions[0]!.options[1]!.label = 'Add a missing receipt assertion'; },
|
||||
]) {
|
||||
const call = handoff();
|
||||
mutate(call);
|
||||
expect(isCeoCompletionHandoff(fingerprint(reanswer(call)))).toBe(false);
|
||||
}
|
||||
const freeform = handoff();
|
||||
freeform.answers![freeform.questions[0]!.question] = 'Please add another test first';
|
||||
expect(isCeoCompletionHandoff(fingerprint(freeform))).toBe(false);
|
||||
});
|
||||
|
||||
test('actual report edits precede the handoff and retain the strict native Exit and freshness checks', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ceo-closed-navigation-'));
|
||||
const report = path.join(dir, 'plan.md');
|
||||
try {
|
||||
fs.writeFileSync(report, captured.reportContent);
|
||||
const reportAt = Date.parse(captured.reportUpdate.at(-1)!.timestamp) / 1000;
|
||||
fs.utimesSync(report, reportAt, reportAt);
|
||||
const transcript = { status: 'ready' as const, calls: calls(), assistantMessages: [],
|
||||
planReadyRequests: structuredClone(captured.planReadyRequests) };
|
||||
const administrative = new Set(transcript.calls.filter(call => isCeoCompletionHandoff(fingerprint(call)))
|
||||
.map(call => `${call.sessionId}:${call.toolUseId}`));
|
||||
const startedAt = Date.parse('2026-09-09T01:46:12Z');
|
||||
expect(administrative.size).toBe(1);
|
||||
expect(hasNativePlanTerminal(transcript, report, startedAt, 'plan_ready')).toBe(false);
|
||||
expect(hasNativePlanTerminal(transcript, report, startedAt, 'plan_ready', administrative)).toBe(true);
|
||||
transcript.planReadyRequests[0]!.failed = true;
|
||||
expect(hasNativePlanTerminal(transcript, report, startedAt, 'plan_ready', administrative)).toBe(false);
|
||||
transcript.planReadyRequests = [];
|
||||
expect(hasNativePlanTerminal(transcript, report, startedAt, 'plan_ready', administrative)).toBe(false);
|
||||
transcript.planReadyRequests = structuredClone(captured.planReadyRequests);
|
||||
transcript.calls.push({ ...structuredClone(transcript.calls[3]!), toolUseId: 'new-real-finding',
|
||||
answeredAt: transcript.calls.at(-1)!.answeredAt });
|
||||
expect(hasNativePlanTerminal(transcript, report, startedAt, 'plan_ready', administrative)).toBe(false);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,120 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { nextCeoModeNavigation } from './helpers/ceo-mode-option';
|
||||
import { planCountQuestionInput } from './helpers/claude-pty-runner';
|
||||
import type { NativePlanQuestionCall } from './helpers/plan-count-transcript';
|
||||
import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles';
|
||||
import calls from './fixtures/ceo-mode-prerequisite-o-calls.json';
|
||||
function ownedIdentity(pid: number): string | null {
|
||||
try { return execFileSync('ps', ['-p', String(pid), '-o', 'lstart=', '-o', 'command='], { encoding: 'utf8', timeout: 5000 }).trim(); }
|
||||
catch { return null; }
|
||||
}
|
||||
function cleanupFake(events: string, fake: string): void {
|
||||
if (!fs.existsSync(events)) return;
|
||||
const first = JSON.parse(fs.readFileSync(events, 'utf8').split('\n')[0]!);
|
||||
if (first.type === 'pid' && Number.isInteger(first.pid) && first.identity?.includes(fake) &&
|
||||
ownedIdentity(first.pid) === first.identity) {
|
||||
try { process.kill(first.pid, 'SIGKILL'); } catch { /* already exited */ }
|
||||
}
|
||||
}
|
||||
function pending(i: number): NativePlanQuestionCall {
|
||||
const { answers, answeredAt, ...call } = structuredClone(calls[i]!.call);
|
||||
return { ...call, answered: false };
|
||||
}
|
||||
function pane(call: NativePlanQuestionCall, index: number): string {
|
||||
const q = call.questions[index]!;
|
||||
const header = call.questions.length > 1 ? '← ' + call.questions.map((q, i) => `${i < index ? '☒' : '☐'} ${q.header}`).join(' ') + ' ✔ Submit →' : '☐ ' + q.header;
|
||||
return `${header}\n${q.question}\n${q.options.map((o, i) => `${i === 0 ? '❯' : ' '} ${i + 1}. ${o.label}`).join('\n')}\nEnter to select · ${call.questions.length > 1 ? 'Tab/Arrow keys' : '↑/↓'} to navigate · Esc to cancel`;
|
||||
}
|
||||
function pick(call: NativePlanQuestionCall, index: number) {
|
||||
const screen = pane(call, index), action = nextCeoModeNavigation(screen, 'SCOPE EXPANSION', new Set(), call);
|
||||
if (action.kind !== 'question') throw new Error(JSON.stringify(action));
|
||||
return { action, input: planCountQuestionInput(screen, action.question, 'index' in action ? action.index as number : 1) };
|
||||
}
|
||||
describe('CEO mode prerequisite navigation', () => {
|
||||
test('both captured expansion attempts select standard review', () => {
|
||||
for (const i of [1, 2]) {
|
||||
const call = pending(i), index = call.questions.findIndex(q => q.header === 'Design doc');
|
||||
const result = pick(call, index);
|
||||
expect(result.action.question.nativeCall).toBe(call);
|
||||
expect(result.action.question.nativeQuestionIndex).toBe(index);
|
||||
expect(result.input).toBe('2');
|
||||
}
|
||||
});
|
||||
test('active tab, offered order, and the actual HOLD setup remain intact', () => {
|
||||
expect(pick(pending(1), 0).input).toBe('1'); expect(pick(pending(1), 1).input).toBe('1');
|
||||
const call = pending(1); call.questions.reverse(); call.questions[0]!.options.reverse();
|
||||
expect(pick(call, 0).input).toBe('1'); expect(pick(pending(0), 2).input).toBe('1');
|
||||
});
|
||||
test('ordinary questions and unrecognized skips keep the existing first choice', () => {
|
||||
for (const [question, labels] of [
|
||||
['Which storage strategy?', ['Server database', 'Local storage']],
|
||||
['No design doc found. Run /office-hours?', ['Run /office-hours first', 'Skip']],
|
||||
['Should the product show office-hours suggestions?', ['Build it', 'Skip — standard review']],
|
||||
] as const) {
|
||||
const call = pending(2); call.questions[0] = {header:'Choice',question,options:labels.map(label=>({label}))};
|
||||
expect(pick(call, 0).input).toBe('1');
|
||||
}
|
||||
});
|
||||
test('redraw dedup and mode targeting remain intact', () => {
|
||||
const call=pending(2),screen=pane(call,0),seen=new Set<string>();
|
||||
expect(nextCeoModeNavigation(screen,'HOLD SCOPE',seen,call).kind).toBe('question');
|
||||
expect(nextCeoModeNavigation(screen,'HOLD SCOPE',seen,call)).toEqual({kind:'wait'});
|
||||
const modes='☐ Review mode\nWhich mode?\n❯ 1. SELECTIVE EXPANSION\n 2. HOLD SCOPE\n 3. SCOPE EXPANSION\n 4. SCOPE REDUCTION';
|
||||
for(const [mode,index] of [['HOLD SCOPE',2],['SCOPE EXPANSION',3]] as const){const a=nextCeoModeNavigation(modes,mode,new Set());expect(a.kind).toBe('mode');if(a.kind==='mode')expect(a.index).toBe(index);}
|
||||
});
|
||||
test('fixture and free regression select only the mode-routing eval', () => {
|
||||
for(const file of ['test/ceo-mode-prerequisite.test.ts','test/fixtures/ceo-mode-prerequisite-o-calls.json'])expect(selectTests([file],E2E_TOUCHFILES).selected).toEqual(['plan-ceo-mode-routing']);
|
||||
});
|
||||
});
|
||||
for(const fixtureIndex of [1,2])test.skipIf(process.platform==='win32')(`fake native PTY skips prerequisite ${fixtureIndex} and confirms target posture`,async()=>{
|
||||
const dir=fs.mkdtempSync(path.join(os.tmpdir(),'ceo-mode-prerequisite-')),fake=path.join(dir,'fake-claude'),events=path.join(dir,'events.jsonl'),worker=path.join(dir,'worker.ts');
|
||||
const root=path.resolve(import.meta.dir,'..'),call=pending(fixtureIndex),screens=call.questions.map((_,i)=>pane(call,i)),expected=call.questions.map(q=>q.header==='Design doc'?'2':'1');
|
||||
fs.writeFileSync(fake,`#!${process.execPath}\n`+`
|
||||
import * as fs from 'node:fs';import * as path from 'node:path';import {execFileSync} from 'node:child_process';
|
||||
const call=${JSON.stringify(call)},screens=${JSON.stringify(screens)},expected=${JSON.stringify(expected)},out=${JSON.stringify(events)};
|
||||
const dir=path.join(process.env.CLAUDE_CONFIG_DIR,'projects','mode-prerequisite');fs.mkdirSync(dir,{recursive:true});
|
||||
const native=(role,content,extra={})=>fs.appendFileSync(path.join(dir,call.sessionId+'.jsonl'),JSON.stringify({cwd:process.cwd(),sessionId:call.sessionId,isSidechain:false,timestamp:new Date().toISOString(),message:{role,content},...extra})+'\\n');
|
||||
const record=e=>fs.appendFileSync(out,JSON.stringify(e)+'\\n');const show=s=>process.stdout.write('\\x1b[2J\\x1b[H'+s.replace(/\\n/g,'\\r\\n'));
|
||||
const mode={header:'Review mode',question:'Which mode?',options:[{label:'SELECTIVE EXPANSION'},{label:'HOLD SCOPE'},{label:'SCOPE EXPANSION'},{label:'SCOPE REDUCTION'}]};
|
||||
let at=0,started=false;const answers={};record({type:'pid',pid:process.pid,identity:execFileSync('ps',['-p',String(process.pid),'-o','lstart=','-o','command='],{encoding:'utf8',timeout:5000}).trim()});process.stdin.setRawMode?.(true);process.stdout.write('FIXTURE_READY');
|
||||
process.stdin.on('data',data=>{const input=data.toString();record({type:'input',input});
|
||||
if(!started){started=true;native('assistant',[{type:'tool_use',id:call.toolUseId,name:'AskUserQuestion',input:{questions:call.questions}}]);show(screens[0]);return;}
|
||||
if(at<expected.length){if(input!==expected[at]){show('Office-hours diversion');return;}answers[call.questions[at].question]=call.questions[at].options[Number(input)-1].label;at++;if(at<expected.length){show(screens[at]);return;}
|
||||
native('user',[{type:'tool_result',tool_use_id:call.toolUseId,content:'Answered.'}],{toolUseResult:{answers}});native('assistant',[{type:'tool_use',id:'mode-choice',name:'AskUserQuestion',input:{questions:[mode]}}]);show('☐ Review mode\\nWhich mode?\\n❯ 1. SELECTIVE EXPANSION\\n 2. HOLD SCOPE\\n 3. SCOPE EXPANSION\\n 4. SCOPE REDUCTION\\nEnter to select · ↑/↓ to navigate · Esc to cancel');return;}
|
||||
if(input!=='3')throw new Error('wrong mode');native('user',[{type:'tool_result',tool_use_id:'mode-choice',content:'Answered.'}],{toolUseResult:{answers:{[mode.question]:'SCOPE EXPANSION'}}});native('assistant',[{type:'text',text:'I will explore expansion opportunities for the saved-view workflow.'}],{timestamp:new Date(Date.now()+2).toISOString()});show('● I will explore expansion opportunities.');});setInterval(()=>{},1000);
|
||||
`,{mode:0o755});
|
||||
const url=(file:string)=>JSON.stringify(pathToFileURL(path.join(root,file)).href);
|
||||
fs.writeFileSync(worker,`
|
||||
import {launchClaudePty,planCountQuestionInput} from ${url('test/helpers/claude-pty-runner.ts')};import {nextCeoModeNavigation,hasNativePostAnswerCeoPosture} from ${url('test/helpers/ceo-mode-option.ts')};import {readPlanCountTranscript} from ${url('test/helpers/plan-count-transcript.ts')};
|
||||
const cwd=${JSON.stringify(dir)},s=await launchClaudePty({cwd,observeScreen:true,timeoutMs:5000}),seen=new Set();let selectedAt=0,matched=false;
|
||||
try{await s.waitFor('FIXTURE_READY',{timeoutMs:2000,pollMs:20});s.send('/plan-ceo-review\\r');for(let i=0;i<160;i++){await Bun.sleep(20);const screen=await s.currentScreen(),t=readPlanCountTranscript(s.hermeticConfigDir,cwd);if(selectedAt&&hasNativePostAnswerCeoPosture(t,'SCOPE EXPANSION',/\\bexpansion\\b/i,selectedAt)){matched=true;break;}const a=nextCeoModeNavigation(screen,'SCOPE EXPANSION',seen,t.calls.find(c=>!c.answered&&!c.failed),s.visibleText());if(a.kind==='question')s.send(planCountQuestionInput(screen,a.question,'index' in a?a.index:1));if(a.kind==='mode'){selectedAt=Date.now();s.send(planCountQuestionInput(screen,a.question,a.index));}}if(!matched)throw new Error('mode posture absent after prerequisite '+s.visibleText());process.stdout.write('native-mode-confirmed');}finally{await s.close();}
|
||||
`);
|
||||
const child=Bun.spawn([process.execPath,worker],{cwd:root,env:{...process.env,BROWSE_TERMINAL_BINARY:fake,EVALS_HERMETIC:'1'},stdout:'pipe',stderr:'pipe'}),timer=setTimeout(()=>child.kill('SIGKILL'),10000);
|
||||
try{const [code,stdout,stderr]=await Promise.all([child.exited,new Response(child.stdout).text(),new Response(child.stderr).text()]);expect(code,stdout+stderr+(fs.existsSync(events)?fs.readFileSync(events,'utf8'):'no fake events')).toBe(0);expect(stdout).toBe('native-mode-confirmed');const rows=fs.readFileSync(events,'utf8').trim().split('\n').map(l=>JSON.parse(l));expect(rows.filter(e=>e.type==='input').map(e=>e.input)).toEqual(['/plan-ceo-review\r',...expected,'3']);expect(()=>process.kill(rows[0].pid,0)).toThrow();}
|
||||
finally{clearTimeout(timer);child.kill('SIGKILL');cleanupFake(events,fake);fs.rmSync(dir,{recursive:true,force:true});}
|
||||
},12000);
|
||||
|
||||
|
||||
test.skipIf(process.platform === 'win32')('outer cleanup requires the recorded fake process identity', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ceo-mode-owned-cleanup-'));
|
||||
const fake = path.join(dir, 'fake.ts'), events = path.join(dir, 'events.jsonl');
|
||||
fs.writeFileSync(fake, 'setInterval(() => {}, 1000);');
|
||||
const child = Bun.spawn([process.execPath, fake], {stdout:'ignore',stderr:'ignore'});
|
||||
try {
|
||||
const identity = ownedIdentity(child.pid);
|
||||
expect(identity?.includes(fake)).toBe(true);
|
||||
fs.writeFileSync(events, JSON.stringify({type:'pid',pid:child.pid,identity:'foreign identity'})+'\n');
|
||||
cleanupFake(events, fake);
|
||||
expect(() => process.kill(child.pid, 0)).not.toThrow();
|
||||
fs.writeFileSync(events, JSON.stringify({type:'pid',pid:child.pid,identity})+'\n');
|
||||
cleanupFake(events, fake);
|
||||
await child.exited;
|
||||
expect(() => process.kill(child.pid, 0)).toThrow();
|
||||
expect(() => cleanupFake(events, fake)).not.toThrow();
|
||||
} finally {child.kill('SIGKILL');fs.rmSync(dir,{recursive:true,force:true});}
|
||||
},5000);
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import captured from './fixtures/devex-review-o-calls.json';
|
||||
import retry from './fixtures/devex-output-o-retry-call.json';
|
||||
import { isDevexReviewIssue } from './helpers/devex-count-fixture';
|
||||
import { nativePlanCallFingerprint } from './helpers/claude-pty-runner';
|
||||
import type { NativePlanQuestionCall } from './helpers/plan-count-transcript';
|
||||
import { E2E_TOUCHFILES } from './helpers/touchfiles';
|
||||
|
||||
const calls = () => structuredClone(captured.calls) as NativePlanQuestionCall[];
|
||||
const fp = (call: NativePlanQuestionCall) => nativePlanCallFingerprint(call, 0, true);
|
||||
describe('documented expected-output gaps are substantive DX decisions', () => {
|
||||
test('all eleven actual O calls retain two setup and nine substantive decisions', () => {
|
||||
const original=calls();
|
||||
expect(original.map(call=>isDevexReviewIssue(fp(call)))).toEqual([false,false,true,true,true,true,true,true,true,true,true]);
|
||||
expect(original).toEqual(calls());
|
||||
// The later migration note and demo exemption are additional offered
|
||||
// changes, not retroactively included in earlier selected options.
|
||||
expect(original[3]!.answers![original[3]!.questions[0]!.question]).toContain('EVALKIT_SKIP_CI_CHECK');
|
||||
expect(original[10]!.questions[0]!.header).toBe('TODO: Demo CI exemption');
|
||||
});
|
||||
|
||||
test('the actual README output decision is counted before or after the review boundary', () => {
|
||||
const fingerprint=fp(calls()[7]!);fingerprint.promptSnippet='Short diagnostic text';
|
||||
for(const preReview of [true,false])expect(isDevexReviewIssue({...fingerprint,preReview})).toBe(true);
|
||||
});
|
||||
|
||||
test('equivalent output-documentation gaps do not depend on an issue number', () => {
|
||||
for(const question of [
|
||||
'The quickstart has no expected output, so developers cannot recognize a successful run.',
|
||||
'Expected output is absent from the documentation. Add an example of a successful command?',
|
||||
'The README does not show the output to expect. Should we document the success signal?',
|
||||
]) {
|
||||
const call=calls()[7]!;call.questions[0]!.header='Documentation gap';call.questions[0]!.question=question;
|
||||
call.answers={[question]:call.questions[0]!.options[0]!.label};expect(isDevexReviewIssue(fp(call))).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('missing answers, confirmation-only options and references to working output do not count', () => {
|
||||
const actual=calls()[7]!;
|
||||
for(const mutate of [
|
||||
(call:NativePlanQuestionCall)=>{call.answered=false;},
|
||||
(call:NativePlanQuestionCall)=>{call.answers={};},
|
||||
(call:NativePlanQuestionCall)=>{call.questions[0]!.header='Empathy check';},
|
||||
(call:NativePlanQuestionCall)=>{call.questions[0]!.options=[{label:'Read the documentation'},{label:'Continue the review'}];},
|
||||
(call:NativePlanQuestionCall)=>{const q=call.questions[0]!;q.question='The README already documents the expected output. Which file should I inspect next?';call.answers={[q.question]:q.options[0]!.label};},
|
||||
(call:NativePlanQuestionCall)=>{const q=call.questions[0]!;q.question='Which documentation should I inspect next?';call.answers={[q.question]:q.options[0]!.label};},
|
||||
]) {const call=structuredClone(actual);mutate(call);expect(isDevexReviewIssue(fp(call))).toBe(false);}
|
||||
const partial=calls()[1]!;partial.questions.push(actual.questions[0]!);partial.unansweredQuestionIndices=[1];
|
||||
expect(isDevexReviewIssue(fp(partial))).toBe(false);
|
||||
});
|
||||
|
||||
test('the actual retry sample-demo output proposal is the same documentation gap', () => {
|
||||
const call=structuredClone(retry.call) as NativePlanQuestionCall;
|
||||
expect(isDevexReviewIssue(fp(call))).toBe(true);
|
||||
expect(call.questions[0]!.options[0]!.label).toContain('Add to plan: include sample demo output in README');
|
||||
for (const question of [
|
||||
'The README shows no example output, so success is unspecified.',
|
||||
'Sample demo output is missing from the quickstart documentation.',
|
||||
]) {const next=structuredClone(call);next.questions[0]!.question=question;next.answers={[question]:next.questions[0]!.options[0]!.label};expect(isDevexReviewIssue(fp(next))).toBe(true);}
|
||||
for (const question of [
|
||||
'The README already shows sample demo output. Which documentation should I read next?',
|
||||
'Should we inspect example output in the README?',
|
||||
'README expected output is not missing.',
|
||||
'README already shows expected output; the missing item is a changelog.',
|
||||
'No expected output is missing from README.',
|
||||
'The README has no missing expected output. Should we show another example?',
|
||||
'No sample demo output is missing from README. Should we show another example?',
|
||||
]) {const next=structuredClone(call);next.questions[0]!.question=question;next.answers={[question]:next.questions[0]!.options[0]!.label};expect(isDevexReviewIssue(fp(next))).toBe(false);}
|
||||
call.questions[0]!.options=[{label:'Read the README'},{label:'Continue the review'}];
|
||||
expect(isDevexReviewIssue(fp(call))).toBe(false);
|
||||
});
|
||||
|
||||
test('the captured documentation regression remains a paid dependency', () => {
|
||||
for(const file of ['test/devex-output-o.test.ts','test/fixtures/devex-review-o-calls.json','test/fixtures/devex-output-o-retry-call.json'])
|
||||
expect(E2E_TOUCHFILES['plan-devex-finding-count']).toContain(file);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import captured from './fixtures/devex-review-o-retry-calls.json';
|
||||
import { isDevexReviewIssue } from './helpers/devex-count-fixture';
|
||||
import { nativePlanCallFingerprint } from './helpers/claude-pty-runner';
|
||||
import type { NativePlanQuestionCall } from './helpers/plan-count-transcript';
|
||||
import { E2E_TOUCHFILES } from './helpers/touchfiles';
|
||||
|
||||
const calls=()=>structuredClone(captured.calls) as NativePlanQuestionCall[];
|
||||
const evaluate=(call:NativePlanQuestionCall)=>isDevexReviewIssue(nativePlanCallFingerprint(call,0,true));
|
||||
const selected=(call:NativePlanQuestionCall,label:string)=>{call.answers={[call.questions[0]!.question]:label};};
|
||||
|
||||
describe('actual repair choices remain substantive within DX setup families',()=>{
|
||||
test('all thirteen retry calls retain five setup, seven substantive and one handoff',()=>{
|
||||
const original=calls();
|
||||
expect(original.map(evaluate)).toEqual([false,false,false,true,true,true,true,true,true,false,false,true,false]);
|
||||
expect(original).toEqual(calls());
|
||||
expect(original[3]!.questions[0]!.header).toBe('TTHW target');
|
||||
expect(original[4]!.questions[0]!.header).toBe('Magical moment');
|
||||
expect(original[9]!.questions[0]!.header).toBe('Confusion report');
|
||||
});
|
||||
|
||||
test('selected CI bypass and new progress feedback are actual offered repairs',()=>{
|
||||
for(const index of [3,4]){
|
||||
const call=calls()[index]!;
|
||||
for(const preReview of [true,false]){
|
||||
const fp=nativePlanCallFingerprint(call,0,preReview);fp.promptSnippet='Short display hint';
|
||||
expect(isDevexReviewIssue(fp)).toBe(true);
|
||||
}
|
||||
expect(call.answers![call.questions[0]!.question]).toContain('add skip flag');
|
||||
}
|
||||
});
|
||||
|
||||
test('pure confirmations, unselected repairs and unrelated premises remain setup',()=>{
|
||||
for(const index of [3,4])for(const mutate of [
|
||||
(call:NativePlanQuestionCall)=>{const q=call.questions[0]!;q.options.unshift({label:'Confirm the already agreed target and vehicle'});selected(call,q.options[0]!.label);},
|
||||
(call:NativePlanQuestionCall)=>{const q=call.questions[0]!;q.options[0]!.label=q.options[0]!.label.replace(/add skip flag[^()]*/i,'keep the already approved behavior ');selected(call,q.options[0]!.label);},
|
||||
(call:NativePlanQuestionCall)=>{const q=call.questions[0]!;q.question='Confirm the settled benchmark and delivery vehicle. <gstack-qid:'+ (index===3?'plan-devex-review-tthw-tier':'plan-devex-review-magical-moment') +'>';selected(call,q.options[0]!.label);},
|
||||
(call:NativePlanQuestionCall)=>{const q=call.questions[0]!;q.question=q.question.replace(/<gstack-qid:[^>]+>/,'<gstack-qid:plan-devex-review-persona>');selected(call,q.options[0]!.label);},
|
||||
]) {const call=calls()[index]!;mutate(call);expect(evaluate(call)).toBe(false);}
|
||||
});
|
||||
|
||||
test('native answer and exact offered choice are mandatory for repair precedence',()=>{
|
||||
for(const index of [3,4])for(const mutate of [
|
||||
(call:NativePlanQuestionCall)=>{call.answered=false;},
|
||||
(call:NativePlanQuestionCall)=>{call.failed=true;},
|
||||
(call:NativePlanQuestionCall)=>{call.answers={};},
|
||||
(call:NativePlanQuestionCall)=>{selected(call,'Invented remedy');},
|
||||
(call:NativePlanQuestionCall)=>{call.unansweredQuestionIndices=[0];},
|
||||
(call:NativePlanQuestionCall)=>{call.questions[0]!.options.push({...call.questions[0]!.options[0]!});},
|
||||
]) {const call=calls()[index]!;mutate(call);expect(evaluate(call)).toBe(false);}
|
||||
for(const index of [3,4]) {
|
||||
const fp=nativePlanCallFingerprint(calls()[index]!,0,true);
|
||||
expect(isDevexReviewIssue({...fp,signature:'foreign:native-call'})).toBe(false);
|
||||
expect(isDevexReviewIssue({...fp,nativeCall:undefined,promptSnippet:'Confirm the already agreed target and vehicle'})).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('captured setup-repair boundaries stay in the paid dependency family',()=>{
|
||||
for(const file of ['test/devex-setup-remedy-o.test.ts','test/fixtures/devex-review-o-retry-calls.json'])
|
||||
expect(E2E_TOUCHFILES['plan-devex-finding-count']).toContain(file);
|
||||
});
|
||||
});
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
⎿ Allowed by auto mode classifier
|
||||
|
||||
● Bash(# Phase 0: UI and DX scope detection
|
||||
PLAN="/tmp/gstack-paid-shard-KqNWnL/tmp/gstack-autoplan-chain-E6SYLb/.claude/plans/ui-heavy-feature.md"…)
|
||||
⎿ === UI SCOPE DETECTION ===
|
||||
20
|
||||
# Plan: User Dashboard Page
|
||||
… +15 lines (ctrl+o to expand)
|
||||
⎿ Allowed by auto mode classifier
|
||||
|
||||
● Now I have all the Phase 0 context. Let me read the CEO phase section, then write the initial plan file structure.
|
||||
|
||||
Read 1 file (ctrl+o to expand)
|
||||
|
||||
● Let me write the initial plan structure and then handle the routing instruction.
|
||||
|
||||
● Updated plan
|
||||
⎿ /plan to preview
|
||||
|
||||
● Now I need to handle the routing instruction that fired at skill start. This is required before continuing with the
|
||||
main review.
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
Planning:
|
||||
/tmp/gstack-paid-shard-KqNWnL/tmp/gstack-hermetic-1924999-9eI1jB/with-skills/.claude/plans/cuddly-sparking-magpie.md
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
☐ Routing rules
|
||||
|
||||
│ gstack works best when your project's CLAUDE.md includes skill routing rules. Add them now?
|
||||
|
||||
❯ 1. Add routing rules (Recommended)
|
||||
Append skill routing rules to CLAUDE.md so Claude automatically invokes the right skill (/autoplan, /ship, /qa,
|
||||
etc.) without you having to type the command. One-time setup, committed to the repo.
|
||||
2. Skip for now
|
||||
Invoke skills manually going forward. You can re-enable later by asking Claude to add routing rules.
|
||||
3. Type something.
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
4. Chat about this
|
||||
|
||||
Enter to select · ↑/↓ to navigate · Esc to cancel
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"sessionId": "fe085d51-f2f2-4d26-9d49-f450afe82ade",
|
||||
"toolUseId": "toolu_01ARahLsLhgiE5Qto8VGoQqg",
|
||||
"answered": false,
|
||||
"failed": false,
|
||||
"questions": [
|
||||
{
|
||||
"question": "gstack works best when your project's CLAUDE.md includes skill routing rules. Add them now?",
|
||||
"header": "Routing setup",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Add routing rules to CLAUDE.md (Recommended)",
|
||||
"description": "Creates CLAUDE.md in the project root with gstack skill routing rules so Claude knows when to invoke each skill automatically. One-time setup, committed to git."
|
||||
},
|
||||
{
|
||||
"label": "No thanks, I'll invoke skills manually",
|
||||
"description": "Skip CLAUDE.md setup — you'll continue invoking skills with /slash commands when needed."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"question": "No design doc found for this branch. /office-hours produces a structured problem statement, premise challenge, and explored alternatives — sharper input for the review. Run it first or proceed with standard review?",
|
||||
"header": "Prerequisite doc",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Skip — proceed with standard review (Recommended)",
|
||||
"description": "The plan is detailed enough (77 lines, clear scope, existing contracts documented) to run the full review pipeline now. /office-hours adds value for exploratory or ambiguous plans."
|
||||
},
|
||||
{
|
||||
"label": "Run /office-hours first",
|
||||
"description": "Produces a structured design doc with premise challenge and alternatives before the review starts. Adds ~10 min but sharpens the CEO phase significantly."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
|
||||
|
||||
Read 1 file, listed 2 directories, ran 2 shell commands
|
||||
|
||||
● Running context recovery and Phase 0 setup.
|
||||
|
||||
Ran 4 shell commands
|
||||
|
||||
● I see the plan. Setting up ACTIVE_PLAN and running scope detection before the routing AskUserQuestion.
|
||||
|
||||
Searched for 1 pattern, ran 1 shell command
|
||||
|
||||
● Codex passes skipped (codex_reviews disabled) — running Claude adversarial only.
|
||||
|
||||
Now writing the ACTIVE_PLAN and handling the routing instruction.
|
||||
|
||||
● Updated plan
|
||||
⎿ /plan to preview
|
||||
|
||||
Ran 1 shell command
|
||||
|
||||
● No design doc found. I'll surface both the routing setup and the prerequisite check in one pass.
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
Planning:
|
||||
/tmp/gstack-paid-shard-KqNWnL/tmp/gstack-hermetic-1924999-9eI1jB/with-skills/.claude/plans/lexical-rolling-boole.md
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
← ☐ Routing setup ☐ Prerequisite doc ✔ Submit →
|
||||
|
||||
│ gstack works best when your project's CLAUDE.md includes skill routing rules. Add them now?
|
||||
|
||||
❯ 1. Add routing rules to CLAUDE.md (Recommended)
|
||||
Creates CLAUDE.md in the project root with gstack skill routing rules so Claude knows when to invoke each skill
|
||||
automatically. One-time setup, committed to git.
|
||||
2. No thanks, I'll invoke skills manually
|
||||
Skip CLAUDE.md setup — you'll continue invoking skills with /slash commands when needed.
|
||||
3. Type something.
|
||||
────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
|
||||
4. Chat about this
|
||||
|
||||
Enter to select · Tab/Arrow keys to navigate · Esc to cancel
|
||||
+232
@@ -0,0 +1,232 @@
|
||||
{
|
||||
"sourceHead": "860f385a82aacb6f83191b1d2ac999cda0b1088a",
|
||||
"sessionId": "d9ca5e3b-d607-48da-ad1d-a0907fee5897",
|
||||
"calls": [
|
||||
{
|
||||
"sessionId": "d9ca5e3b-d607-48da-ad1d-a0907fee5897",
|
||||
"toolUseId": "toolu_01UC1urBF2UDGGJAeTiuWAwt",
|
||||
"questions": [
|
||||
{
|
||||
"question": "No design doc found for this branch. `/office-hours` produces a structured problem statement, premise challenge, and explored alternatives \u2014 it gives this review much sharper input to work with. Takes about 10 minutes. The design doc is per-feature, not per-product \u2014 it captures the thinking behind this specific change. Want to run it first? <gstack-qid:ceo-review-prereq-office-hours>",
|
||||
"header": "Prereq skill",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Skip \u2014 proceed with standard review (Recommended)",
|
||||
"description": "The plan is specific enough to review directly. The two test cases are clearly scoped and the existing infra is documented in the plan."
|
||||
},
|
||||
{
|
||||
"label": "Run /office-hours first",
|
||||
"description": "Produce a structured design doc first \u2014 problem statement, alternatives considered, constraints \u2014 then continue the CEO review with that as input."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"No design doc found for this branch. `/office-hours` produces a structured problem statement, premise challenge, and explored alternatives \u2014 it gives this review much sharper input to work with. Takes about 10 minutes. The design doc is per-feature, not per-product \u2014 it captures the thinking behind this specific change. Want to run it first? <gstack-qid:ceo-review-prereq-office-hours>": "Skip \u2014 proceed with standard review (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T01:49:28.927Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "d9ca5e3b-d607-48da-ad1d-a0907fee5897",
|
||||
"toolUseId": "toolu_01Wj1h2nukbgRECCCk52DwNQ",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D1 \u2014 Which implementation approach for the two processPayment() unit tests? <gstack-qid:ceo-test-coverage-approach>\n\nELI10: The plan calls for 2 tests. The question is whether we nail the assertion detail (what exactly does 'correct receipt' mean? what does 'fails clean' look like?) inside those 2 tests, or expand to cover additional error paths at the processPayment() level.\n\nStakes if we pick wrong: Under-specified assertions pass on hollow mocks; over-specified assertions become brittle maintenance burden.\n\nRecommendation: B because explicit assertions make the test a real safety net, not a checkbox.\nCompleteness: A=6/10, B=8/10, C=9/10",
|
||||
"header": "Approach",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "A: As-planned, minimal assertions",
|
||||
"description": "Write the 2 tests with straightforward assertions (receipt exists and charge was called). Fast, but assertions may be too shallow to catch subtle receipt bugs."
|
||||
},
|
||||
{
|
||||
"label": "B: As-planned, explicit assertion specs (Recommended)",
|
||||
"description": "Same 2 tests, but the plan explicitly specifies what 'correct receipt' means (which fields, which values) and what 'fails clean' means (which exception, which log). No extra test count, but real safety nets."
|
||||
},
|
||||
{
|
||||
"label": "C: Extended \u2014 4 tests (add 402 + 429 at processPayment() level)",
|
||||
"description": "Add card-decline (402) and rate-limit (429) tests at the processPayment() level, symmetric with adapter suite coverage. Only valuable if processPayment() has its own 402/429 handling logic beyond what the adapter already handles."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D1 \u2014 Which implementation approach for the two processPayment() unit tests? <gstack-qid:ceo-test-coverage-approach>\n\nELI10: The plan calls for 2 tests. The question is whether we nail the assertion detail (what exactly does 'correct receipt' mean? what does 'fails clean' look like?) inside those 2 tests, or expand to cover additional error paths at the processPayment() level.\n\nStakes if we pick wrong: Under-specified assertions pass on hollow mocks; over-specified assertions become brittle maintenance burden.\n\nRecommendation: B because explicit assertions make the test a real safety net, not a checkbox.\nCompleteness: A=6/10, B=8/10, C=9/10": "A: As-planned, minimal assertions"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T01:50:29.137Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "d9ca5e3b-d607-48da-ad1d-a0907fee5897",
|
||||
"toolUseId": "toolu_01VKuhQqyP3YMx9Y8qr4AkJB",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D2 \u2014 Which review mode for this plan? <gstack-qid:ceo-review-mode>\n\nELI10: This plan adds two unit tests to an existing payment function. The tests are clearly scoped and the infrastructure already exists. The question is whether we hold exactly this scope and make it bulletproof, or surface additional test opportunities for you to cherry-pick.\n\nStakes if we pick wrong: HOLD misses real coverage gaps worth surfacing; SELECTIVE EXPANSION over-proposes and wastes your time on a focused PR.\n\nRecommendation: HOLD SCOPE because this is a targeted test coverage PR with clear, bounded scope \u2014 the right job is to make it bulletproof, not expand it.\n\nNote: options differ in kind, not coverage \u2014 no completeness score.",
|
||||
"header": "Review mode",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "HOLD SCOPE \u2014 bulletproof the plan (Recommended)",
|
||||
"description": "The two tests are the right scope. Review for correctness, assertion quality, edge cases, and observability. No expansions proposed."
|
||||
},
|
||||
{
|
||||
"label": "SELECTIVE EXPANSION \u2014 hold scope + cherry-pick extras",
|
||||
"description": "Start from the two-test baseline, then surface additional coverage opportunities (402, 429, receipt validation depth) individually for you to opt in or out."
|
||||
},
|
||||
{
|
||||
"label": "SCOPE REDUCTION \u2014 find the minimum viable test",
|
||||
"description": "Challenge whether both tests are needed now, or if one is more critical. For teams with very constrained PR review bandwidth."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D2 \u2014 Which review mode for this plan? <gstack-qid:ceo-review-mode>\n\nELI10: This plan adds two unit tests to an existing payment function. The tests are clearly scoped and the infrastructure already exists. The question is whether we hold exactly this scope and make it bulletproof, or surface additional test opportunities for you to cherry-pick.\n\nStakes if we pick wrong: HOLD misses real coverage gaps worth surfacing; SELECTIVE EXPANSION over-proposes and wastes your time on a focused PR.\n\nRecommendation: HOLD SCOPE because this is a targeted test coverage PR with clear, bounded scope \u2014 the right job is to make it bulletproof, not expand it.\n\nNote: options differ in kind, not coverage \u2014 no completeness score.": "HOLD SCOPE \u2014 bulletproof the plan (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T01:50:57.255Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "d9ca5e3b-d607-48da-ad1d-a0907fee5897",
|
||||
"toolUseId": "toolu_016CgrX8J6H7qtj8cdxJAp1V",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D3 \u2014 Section 6 (Tests): '502-exhaustion' test asserts 'fails clean' \u2014 what does that mean? <gstack-qid:ceo-test-fails-clean-spec>\n\nELI10: The plan says the 502-exhaustion test should assert 'retry-with-backoff fires once, then fails clean.' But 'fails clean' is ambiguous. A test that only checks retry count (2 Stripe calls) will PASS even if processPayment() silently swallows the error and returns nil. That's the worst payment failure mode: money attempted, caller has no idea it failed.\n\nStakes if we pick wrong: A test that doesn't assert the failure is observable gives false confidence. The 502-exhaustion test would pass on broken code.\n\nRecommendation: A because payment code must never fail silently \u2014 the test must prove the failure signal reaches the caller.\nCompleteness: A=8/10, B=7/10, C=6/10\nPros/cons:\nA) Assert named observable failure (exception raised OR error value returned)\n \u2705 Guarantees the caller receives a failure signal \u2014 eliminates the silent-swallow failure mode\n \u2705 Works regardless of whether the codebase uses exceptions or error values\n \u274c Requires the implementer to look up the specific exception/return type from the codebase\nB) Assert no exception propagates (processPayment() handles internally, returns nil/error struct)\n \u2705 Simpler assertion in languages that prefer error returns over exceptions\n \u274c Only valid if processPayment() is documented to return an error value \u2014 wrong if it raises\nC) Leave as-is: trust implementer to infer 'fails clean'\n \u2705 No plan change needed\n \u274c Silent failure risk: implementer may write assert(result.nil?) which passes on swallowed errors\nNet: choosing C accepts the risk that the test checks retry count but not failure observability.",
|
||||
"header": "Assertion spec",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "A: Plan states failure must be observable to caller (Recommended)",
|
||||
"description": "Add to plan: 'fails clean' means the caller receives a named failure signal \u2014 either an exception or an explicit error return. The test asserts this signal, not just that 2 Stripe calls occurred."
|
||||
},
|
||||
{
|
||||
"label": "B: Plan states no exception propagates (returns error value)",
|
||||
"description": "Add to plan: processPayment() handles the exhaustion internally and returns nil or an error struct. Test asserts the non-exceptional return with an error indicator."
|
||||
},
|
||||
{
|
||||
"label": "C: Leave as-is \u2014 trust the implementer",
|
||||
"description": "Keep 'fails clean' as the sole spec. Accepts the risk that the test may not verify the failure is visible to the caller."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D3 \u2014 Section 6 (Tests): '502-exhaustion' test asserts 'fails clean' \u2014 what does that mean? <gstack-qid:ceo-test-fails-clean-spec>\n\nELI10: The plan says the 502-exhaustion test should assert 'retry-with-backoff fires once, then fails clean.' But 'fails clean' is ambiguous. A test that only checks retry count (2 Stripe calls) will PASS even if processPayment() silently swallows the error and returns nil. That's the worst payment failure mode: money attempted, caller has no idea it failed.\n\nStakes if we pick wrong: A test that doesn't assert the failure is observable gives false confidence. The 502-exhaustion test would pass on broken code.\n\nRecommendation: A because payment code must never fail silently \u2014 the test must prove the failure signal reaches the caller.\nCompleteness: A=8/10, B=7/10, C=6/10\nPros/cons:\nA) Assert named observable failure (exception raised OR error value returned)\n \u2705 Guarantees the caller receives a failure signal \u2014 eliminates the silent-swallow failure mode\n \u2705 Works regardless of whether the codebase uses exceptions or error values\n \u274c Requires the implementer to look up the specific exception/return type from the codebase\nB) Assert no exception propagates (processPayment() handles internally, returns nil/error struct)\n \u2705 Simpler assertion in languages that prefer error returns over exceptions\n \u274c Only valid if processPayment() is documented to return an error value \u2014 wrong if it raises\nC) Leave as-is: trust implementer to infer 'fails clean'\n \u2705 No plan change needed\n \u274c Silent failure risk: implementer may write assert(result.nil?) which passes on swallowed errors\nNet: choosing C accepts the risk that the test checks retry count but not failure observability.": "A: Plan states failure must be observable to caller (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T01:53:51.547Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "d9ca5e3b-d607-48da-ad1d-a0907fee5897",
|
||||
"toolUseId": "toolu_011vNUcryYgv4UbZyPjnMWnL",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D4 \u2014 Section 6 (Tests): Happy path asserts 'correct receipt is generated' \u2014 what's the minimum assertion? <gstack-qid:ceo-test-receipt-spec>\n\nELI10: 'Correct receipt is generated' could mean anything from 'receipt != nil' to 'every field matches the Stripe charge response.' With Approach A (minimal assertions), we need to agree on the floor. Without a floor, the test can pass with an empty receipt object.\n\nStakes if we pick wrong: Too loose and the test catches nothing useful; too strict and it becomes brittle maintenance.\n\nRecommendation: A because it sets a meaningful floor without over-specifying field-level content (which the receipt-builder tests already cover).\nCompleteness: A=7/10, B=6/10, C=8/10\nPros/cons:\nA) Minimal floor: receipt != nil AND Stripe mock called exactly once\n \u2705 Verifies processPayment() produced something and charged Stripe exactly once\n \u2705 Not brittle \u2014 doesn't depend on receipt field names or values\n \u274c Doesn't catch a receipt built from the wrong charge (e.g., wrong amount)\nB) Leave as-is: 'correct receipt' is sufficient guidance\n \u2705 No plan change needed\n \u274c Implementer may write assert(receipt) which passes on a stub object with no real content\nC) Add Stripe charge ID linkage: receipt != nil AND receipt.stripe_charge_id == mock_charge_id AND Stripe called once\n \u2705 Proves the receipt was built from THIS charge, not a stale one\n \u274c Ties the test to the receipt's field name \u2014 more brittle than A if receipt schema changes\nNet: A sets a real floor without being fragile. C is better but slightly contradicts Approach A (minimal).",
|
||||
"header": "Receipt assertion",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "A: receipt != nil AND Stripe called exactly once (Recommended)",
|
||||
"description": "Plan specifies: the happy path test asserts receipt is non-nil and the Stripe mock shows exactly one charge call. Minimal but real signal."
|
||||
},
|
||||
{
|
||||
"label": "B: Leave as-is \u2014 trust the implementer",
|
||||
"description": "Keep 'correct receipt is generated' as the sole spec. Faster, but the test floor is undefined."
|
||||
},
|
||||
{
|
||||
"label": "C: receipt != nil AND Stripe called once AND charge ID matches",
|
||||
"description": "Plan specifies: receipt is non-nil, Stripe called once, and receipt.stripe_charge_id equals the mock charge ID. Stronger but slightly more brittle."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D4 \u2014 Section 6 (Tests): Happy path asserts 'correct receipt is generated' \u2014 what's the minimum assertion? <gstack-qid:ceo-test-receipt-spec>\n\nELI10: 'Correct receipt is generated' could mean anything from 'receipt != nil' to 'every field matches the Stripe charge response.' With Approach A (minimal assertions), we need to agree on the floor. Without a floor, the test can pass with an empty receipt object.\n\nStakes if we pick wrong: Too loose and the test catches nothing useful; too strict and it becomes brittle maintenance.\n\nRecommendation: A because it sets a meaningful floor without over-specifying field-level content (which the receipt-builder tests already cover).\nCompleteness: A=7/10, B=6/10, C=8/10\nPros/cons:\nA) Minimal floor: receipt != nil AND Stripe mock called exactly once\n \u2705 Verifies processPayment() produced something and charged Stripe exactly once\n \u2705 Not brittle \u2014 doesn't depend on receipt field names or values\n \u274c Doesn't catch a receipt built from the wrong charge (e.g., wrong amount)\nB) Leave as-is: 'correct receipt' is sufficient guidance\n \u2705 No plan change needed\n \u274c Implementer may write assert(receipt) which passes on a stub object with no real content\nC) Add Stripe charge ID linkage: receipt != nil AND receipt.stripe_charge_id == mock_charge_id AND Stripe called once\n \u2705 Proves the receipt was built from THIS charge, not a stale one\n \u274c Ties the test to the receipt's field name \u2014 more brittle than A if receipt schema changes\nNet: A sets a real floor without being fragile. C is better but slightly contradicts Approach A (minimal).": "A: receipt != nil AND Stripe called exactly once (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T01:54:09.631Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "d9ca5e3b-d607-48da-ad1d-a0907fee5897",
|
||||
"toolUseId": "toolu_01PRKn1JRj5MmrSrq1YgYcwf",
|
||||
"questions": [
|
||||
{
|
||||
"question": "CEO review done. What's next? <gstack-qid:ceo-review-next-steps>\n\nCEO review: CLEARED (HOLD SCOPE, 0 unresolved decisions, 0 critical gaps). Two implementation tasks: T1 (happy path test) and T2 (502-exhaustion test with named observable failure assertion). Eng review is the required shipping gate.",
|
||||
"header": "Next review",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Run /plan-eng-review (Recommended)",
|
||||
"description": "Architecture, code quality, and test completeness audit \u2014 the required gate before shipping this PR."
|
||||
},
|
||||
{
|
||||
"label": "Skip \u2014 handle reviews manually",
|
||||
"description": "Proceed to implementation now and run eng review later, or skip it entirely."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"CEO review done. What's next? <gstack-qid:ceo-review-next-steps>\n\nCEO review: CLEARED (HOLD SCOPE, 0 unresolved decisions, 0 critical gaps). Two implementation tasks: T1 (happy path test) and T2 (502-exhaustion test with named observable failure assertion). Eng review is the required shipping gate.": "Run /plan-eng-review (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T01:57:25.730Z"
|
||||
}
|
||||
],
|
||||
"reportContent": "# CEO Plan Review: Payment Processing \u2014 Test Coverage\n\n## Context\n\nThe plan proposes adding two missing unit tests for `processPayment()`:\n1. Happy path \u2014 successful Stripe charge \u2192 correct receipt generated\n2. 502 exhaustion path \u2014 Stripe returns 502, retry-with-backoff fires once, then fails clean\n\nProduction behavior is untouched. Existing test infrastructure (test factory, virtual sleeper,\nStripe mock, adapter regression suites) is reused. This is a test coverage gap closure,\nnot a new feature.\n\n## System Audit\n\n- Repo: single \"Seed review plan\" commit; only PLAN.md and CLAUDE.md in tree\n- No TODOs/FIXMEs, no prior reviews, no stash\n- No design doc, no handoff note\n- Existing infrastructure per plan: test factory (max_retries=1), virtual sleeper,\n Stripe mock call history, adapter regression suite, receipt-builder regression tests\n- Platform: no remote (local repo)\n\n## Step 0 Decisions\n\n- **Prerequisite**: office-hours skipped (plan is self-contained)\n- **Approach**: A \u2014 as-planned, 2 tests, minimal assertions (Completeness: 6/10)\n - `gstack-shortcut(dec-60b9e22c): ceiling 8/10 \u2014 explicit receipt field assertions + named exception for 502; upgrade when receipt bugs found in prod or refactor breaks receipt shape`\n- **Mode**: HOLD SCOPE \u2014 make the two-test plan bulletproof, no expansion\n\n---\n\n## System Architecture\n\n```\nprocessPayment() [EXISTING \u2014 production code, untouched]\n \u2502\n \u251c\u2500\u2500 Stripe Adapter [EXISTING]\n \u2502 \u251c\u2500\u2500 Timeouts \u2192 tested in adapter suite\n \u2502 \u251c\u2500\u2500 402 \u2192 tested in adapter suite\n \u2502 \u251c\u2500\u2500 429 \u2192 tested in adapter suite\n \u2502 \u2514\u2500\u2500 502+retry \u2192 tested in adapter suite\n \u2502\n \u2514\u2500\u2500 Receipt Builder [EXISTING]\n \u2514\u2500\u2500 failure behavior \u2192 tested in receipt regression suite\n\nTEST INFRASTRUCTURE [EXISTING \u2014 all reused by this plan]\n \u251c\u2500\u2500 Payment test factory (max_retries=1, exposes mock call history)\n \u251c\u2500\u2500 Virtual sleeper (records backoff without real delays)\n \u2514\u2500\u2500 Stripe mock (configurable responses per test)\n\nNEW TESTS [this plan]\n \u251c\u2500\u2500 Happy path\n \u2502 Input: test factory + mock configured for success\n \u2502 Assert: \"correct receipt generated\" \u2190 UNDERSPECIFIED (finding D3)\n \u2502 Charge: Stripe called exactly once \u2190 inferable from mock call history\n \u2514\u2500\u2500 502-exhaustion\n Input: test factory (max_retries=1) + mock returning 502\n Backoff: virtual sleeper records it\n Retries: exactly 2 charge attempts (1 + 1 retry)\n Assert: retry-with-backoff fires once, then \"fails clean\" \u2190 UNDERSPECIFIED (finding D4)\n```\n\n**Data flow \u2014 happy path (4 paths):**\n```\n INPUT \u2500\u2500\u25b6 [Stripe mock: success] \u2500\u2500\u25b6 processPayment() \u2500\u2500\u25b6 Receipt \u2500\u2500\u25b6 OUTPUT\n \u2502 \u2502 \u2502 \u2502 \u2502\n \u25bc \u25bc \u25bc \u25bc \u25bc\n [nil?] [mock bypass?] [builder fail?] [nil check?] [correct?]\n [empty?] [call count?] [partial?] [field check]\n```\n\n**Data flow \u2014 502-exhaustion (4 paths):**\n```\n INPUT \u2500\u2500\u25b6 [Stripe mock: 502] \u2500\u2500\u25b6 processPayment() \u2500\u2500\u25b6 [backoff] \u2500\u2500\u25b6 retry \u2500\u2500\u25b6 [Stripe mock: 502]\n \u2502 \u2502 \u2502 \u2502\n \u25bc \u25bc \u25bc \u25bc\n [2 call attempts] [retry logic] [virtual sleeper] \"fails clean\"\n [UNDEFINED]\n```\n\n---\n\n## Section-by-Section Review (HOLD SCOPE)\n\n### Section 1: Architecture \u2014 No issues\n\nTest infrastructure reuse is clean. Dependency graph is unchanged (tests only). No coupling introduced. No production code touched. Rollback: delete the two test files.\n\n### Section 2: Error & Rescue Map\n\n```\nMETHOD/CODEPATH | WHAT CAN GO WRONG | GAP?\n-------------------------|--------------------------------|------\nprocessPayment() test 1 | Receipt is nil | Partially \u2014 Approach A accepts minimal\n(happy path) | Stripe mock call count wrong | OK \u2014 mock history exposed\n | Receipt fields wrong | Partial \u2014 \"correct\" undefined\n\nprocessPayment() test 2 | Retries exhaust but Line truncated
|
||||
"reportUpdate": [
|
||||
{
|
||||
"toolUseId": "toolu_01Q8xGfdpmzBCB13KyAuSmkW",
|
||||
"name": "Write",
|
||||
"timestamp": "2026-09-09T01:49:09.420Z"
|
||||
},
|
||||
{
|
||||
"toolUseId": "toolu_01TUi3Bzigv43JjoMYABUa1h",
|
||||
"name": "Edit",
|
||||
"timestamp": "2026-09-09T01:53:19.934Z"
|
||||
},
|
||||
{
|
||||
"toolUseId": "toolu_01UociV1iB8VjFjDavk4M8XR",
|
||||
"name": "Edit",
|
||||
"timestamp": "2026-09-09T01:55:28.088Z"
|
||||
},
|
||||
{
|
||||
"toolUseId": "toolu_01N3E3KretonGAQjirYgCHRk",
|
||||
"name": "Edit",
|
||||
"timestamp": "2026-09-09T01:56:39.999Z"
|
||||
}
|
||||
],
|
||||
"planReadyRequests": [
|
||||
{
|
||||
"sessionId": "d9ca5e3b-d607-48da-ad1d-a0907fee5897",
|
||||
"toolUseId": "toolu_011tKAiNhjeiYLsb7PHT3teD",
|
||||
"timestamp": "2026-09-09T01:57:34.490Z",
|
||||
"failed": false
|
||||
}
|
||||
],
|
||||
"historicalRaw": {
|
||||
"step0Count": 3,
|
||||
"reviewCount": 3,
|
||||
"administrativeCount": 0,
|
||||
"state": "in_progress"
|
||||
},
|
||||
"provenance": {
|
||||
"nativeSha256": "f073714b11ccc7976211f1b35db3617a761d6d024f6cdea95698a95acb616815",
|
||||
"originalReportPath": "/tmp/gstack-paid-shard-gH7gOc/tmp/gstack-e2e-plan-ceo-paired-2LWNKB/gstack-test-plan-ceo-paired.md",
|
||||
"firstSeenAt": "2026-09-09T02:01:42.844425+00:00"
|
||||
}
|
||||
}
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
[
|
||||
{
|
||||
"case": "hold-control",
|
||||
"nativeSourceSha256": "175ddd9b387d5d45dd49952a6822ed2eab92a4a66b3b2872e93492b304414009",
|
||||
"requestLine": 71,
|
||||
"resultLine": 72,
|
||||
"call": {
|
||||
"sessionId": "fb129c7f-0801-4e9f-8b63-f44f719b8dec",
|
||||
"toolUseId": "toolu_01DWZHwQ6kuttZAmi2g94NAN",
|
||||
"requestedAt": "2026-09-09T01:52:51.586Z",
|
||||
"answered": true,
|
||||
"answeredAt": "2026-09-09T01:53:04.476Z",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Add gstack skill routing rules to this project's CLAUDE.md?",
|
||||
"header": "Skill routing",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Add routing rules (Recommended)",
|
||||
"description": "Append skill routing rules to CLAUDE.md so Claude automatically invokes the right gstack skill for common requests (strategy, architecture, QA, shipping). One-time setup, committed to the repo."
|
||||
},
|
||||
{
|
||||
"label": "No thanks",
|
||||
"description": "Skip \u2014 I'll invoke skills manually. (This prompt won't reappear once declined.)"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"question": "Enable cross-project learnings? gstack can apply patterns it learned from your other local projects to this review.",
|
||||
"header": "Learnings scope",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Enable cross-project (Recommended)",
|
||||
"description": "Learnings from all your local projects inform this review. Stays fully local \u2014 nothing leaves your machine. Best for solo devs."
|
||||
},
|
||||
{
|
||||
"label": "Project-scoped only",
|
||||
"description": "Only learnings from this specific project apply. Better if you work across multiple client codebases where cross-contamination would be a concern."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"question": "No design doc found for this branch. Run /office-hours first to produce a structured problem statement and explored alternatives \u2014 that gives this CEO review sharper input. (~10 min)",
|
||||
"header": "Prereq skill",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Skip \u2014 review PLAN.md as-is (Recommended)",
|
||||
"description": "Proceed directly with the plan as written. I'll surface gaps and missing context during the review itself."
|
||||
},
|
||||
{
|
||||
"label": "Run /office-hours first",
|
||||
"description": "Produces a design doc with problem statement, alternatives explored, and constraints. Gives the CEO review stronger grounding \u2014 worth it for high-stakes features."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answers": {
|
||||
"Add gstack skill routing rules to this project's CLAUDE.md?": "Add routing rules (Recommended)",
|
||||
"Enable cross-project learnings? gstack can apply patterns it learned from your other local projects to this review.": "Enable cross-project (Recommended)",
|
||||
"No design doc found for this branch. Run /office-hours first to produce a structured problem statement and explored alternatives \u2014 that gives this CEO review sharper input. (~10 min)": "Skip \u2014 review PLAN.md as-is (Recommended)"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "expansion-first",
|
||||
"nativeSourceSha256": "3a810ccebf1241b0639bd613b7d0e2445d1184a565df21c08cc253e4a48b4e8a",
|
||||
"requestLine": 67,
|
||||
"resultLine": 68,
|
||||
"call": {
|
||||
"sessionId": "0b5caec7-417b-415d-9ede-8cdffc960a3f",
|
||||
"toolUseId": "toolu_01UukKj3xcwtCLg5CJBn8C7s",
|
||||
"requestedAt": "2026-09-09T01:53:37.436Z",
|
||||
"answered": true,
|
||||
"answeredAt": "2026-09-09T01:53:50.605Z",
|
||||
"questions": [
|
||||
{
|
||||
"question": "gstack works best when your project's CLAUDE.md includes skill routing rules (e.g. 'strategy \u2192 /plan-ceo-review'). This adds ~10 lines that auto-suggest skills in future sessions. Note: we're in plan mode so the edit + commit will run after plan mode exits.",
|
||||
"header": "Skill routing",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Add routing rules (recommended)",
|
||||
"description": "Append gstack routing rules to CLAUDE.md and commit after plan mode exits. One-time setup, no ongoing cost."
|
||||
},
|
||||
{
|
||||
"label": "Skip",
|
||||
"description": "Keep invoking skills manually. No change to CLAUDE.md."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"question": "gstack can search learnings from your other projects on this machine to find patterns that may apply here. Stays local \u2014 no data leaves your machine. Good for solo devs; skip if you work across client codebases where cross-pollination would be a problem.",
|
||||
"header": "Learnings scope",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Enable cross-project (recommended)",
|
||||
"description": "Search learnings across all local projects. Useful for solo developers or teams on a shared machine."
|
||||
},
|
||||
{
|
||||
"label": "Project-scoped only",
|
||||
"description": "Only surface learnings from this project."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"question": "No design doc found for this branch. /office-hours produces a structured problem statement, premise challenge, and explored alternatives \u2014 it gives this CEO review sharper input. Takes about 10 minutes. The design doc is per-feature, not per-product.",
|
||||
"header": "Design doc",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Run /office-hours first (recommended)",
|
||||
"description": "Generate a design doc now, then the CEO review picks up right after with much richer context."
|
||||
},
|
||||
{
|
||||
"label": "Skip \u2014 standard review",
|
||||
"description": "Proceed without a design doc. Review will still be thorough but works from the plan alone."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answers": {
|
||||
"gstack works best when your project's CLAUDE.md includes skill routing rules (e.g. 'strategy \u2192 /plan-ceo-review'). This adds ~10 lines that auto-suggest skills in future sessions. Note: we're in plan mode so the edit + commit will run after plan mode exits.": "Add routing rules (recommended)",
|
||||
"gstack can search learnings from your other projects on this machine to find patterns that may apply here. Stays local \u2014 no data leaves your machine. Good for solo devs; skip if you work across client codebases where cross-pollination would be a problem.": "Enable cross-project (recommended)",
|
||||
"No design doc found for this branch. /office-hours produces a structured problem statement, premise challenge, and explored alternatives \u2014 it gives this CEO review sharper input. Takes about 10 minutes. The design doc is per-feature, not per-product.": "Run /office-hours first (recommended)"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "expansion-retry",
|
||||
"nativeSourceSha256": "c7b1b01f934321a5744f0443603df9bccd694419d3346c99178a1732d56133b1",
|
||||
"requestLine": 62,
|
||||
"resultLine": 63,
|
||||
"call": {
|
||||
"sessionId": "428232f1-d55d-42ac-8062-ec615d2923f9",
|
||||
"toolUseId": "toolu_01FLL8tiDfw9uJRTGTpQaWgv",
|
||||
"requestedAt": "2026-09-09T01:59:14.794Z",
|
||||
"answered": true,
|
||||
"answeredAt": "2026-09-09T01:59:15.628Z",
|
||||
"questions": [
|
||||
{
|
||||
"question": "No design doc found for this branch. `/office-hours` produces a structured problem statement, premise challenge, and explored alternatives \u2014 it gives this review much sharper input. Want to run it first?",
|
||||
"header": "Design doc",
|
||||
"options": [
|
||||
{
|
||||
"label": "Run /office-hours first (Recommended)",
|
||||
"description": "Takes ~10 minutes. Produces a structured design doc with problem framing, alternatives considered, and key decisions \u2014 makes every finding in this CEO review sharper and better grounded."
|
||||
},
|
||||
{
|
||||
"label": "Skip \u2014 proceed with standard review",
|
||||
"description": "Proceed directly to the CEO review with the plan text as-is. The review will still be rigorous but without a design doc some premise questions will need to be answered inline."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answers": {
|
||||
"No design doc found for this branch. `/office-hours` produces a structured problem statement, premise challenge, and explored alternatives \u2014 it gives this review much sharper input. Want to run it first?": "Run /office-hours first (Recommended)"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
+566
@@ -0,0 +1,566 @@
|
||||
{
|
||||
"sourceSession": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"nativeSha256": "fbec1ec6cb1374a352b83dd32d48f619cf08f214dabd5b9f7f9a1c15c42ea7bc",
|
||||
"transcript": {
|
||||
"status": "ready",
|
||||
"calls": [
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01PdoP9S1U1xRaUyCuaXrteE",
|
||||
"questions": [
|
||||
{
|
||||
"question": "No design doc found for this branch. `/office-hours` produces a structured problem statement, premise challenge, and explored alternatives \u2014 it gives this review much sharper input. That said, EvalKit's README + docs/ are unusually complete: persona, TTHW target, competitive benchmark, and demo delivery vehicle are all pre-decided.\n\nShould I run /office-hours first, or proceed with the existing docs as context?\n\n<gstack-qid:plan-devex-review-prereq-skill>",
|
||||
"header": "Prereq skill",
|
||||
"options": [
|
||||
{
|
||||
"label": "Skip \u2014 proceed with existing docs (Recommended)",
|
||||
"description": "README + benchmarks.md + current-contracts.md + api.md already answer the core design questions. Proceed directly to DX review."
|
||||
},
|
||||
{
|
||||
"label": "Run /office-hours first",
|
||||
"description": "Produces a structured problem statement and surfaces assumptions before the review begins. Takes ~10 minutes."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"No design doc found for this branch. `/office-hours` produces a structured problem statement, premise challenge, and explored alternatives \u2014 it gives this review much sharper input. That said, EvalKit's README + docs/ are unusually complete: persona, TTHW target, competitive benchmark, and demo delivery vehicle are all pre-decided.\n\nShould I run /office-hours first, or proceed with the existing docs as context?\n\n<gstack-qid:plan-devex-review-prereq-skill>": "Skip \u2014 proceed with existing docs (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T01:59:50.425Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_017zNcFTommXjSJMZxARcfZ3",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D1 \u2014 Who is your target developer?\n\nProject/branch: EvalKit SDK 2.0.0b1 beta release polish / main\n\nELI10: Different developers have radically different expectations for SDKs. An ML researcher wants speed to first eval. A platform engineer vets reliability and CI integration. A team lead evaluates upgrade safety. Getting this wrong means the whole review optimizes for the wrong person.\n\nStakes if we pick wrong: We polish the wrong touchpoints. A platform engineer's concerns (SLA docs, SBOM) get solved while an ML engineer's pain (5-min CI block, auth error, missing quickstart) stays broken.\n\nRecommendation: A because README explicitly identifies this persona and the onboarding study was run with them in mind.\n\nCompleteness: Note: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) ML engineer / Python daily driver (recommended)\n \u2714 Matches the README persona exactly: writes Python daily, uses terminal, wants local result before CI\n \u2714 Onboarding study was run with this persona; benchmark data is directly applicable\n \u274c May miss concerns of platform teams who deploy EvalKit in shared CI pipelines\nB) Platform / DevOps engineer\n \u2714 Covers reliability, SLA, and enterprise CI integration concerns that ML engineers may not raise\n \u274c Not the stated persona in README; polishing for them first would misalign the review\nC) Data scientist / less Python-fluent\n \u2714 Would catch gaps in error message clarity and conceptual scaffolding\n \u274c Over-indexes on hand-holding for an SDK whose README assumes daily Python usage\n\nNet: Confirming the persona anchors every friction-point recommendation to the person who will actually be doing the onboarding.\n\n<gstack-qid:plan-devex-review-persona>",
|
||||
"header": "Persona",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) ML engineer / Python daily driver (Recommended)",
|
||||
"description": "Writes Python daily, uses terminal, wants local result before connecting SDK to production CI."
|
||||
},
|
||||
{
|
||||
"label": "B) Platform / DevOps engineer",
|
||||
"description": "Evaluates reliability, SLA, enterprise CI integration, and shared infra concerns."
|
||||
},
|
||||
{
|
||||
"label": "C) Data scientist / less Python-fluent",
|
||||
"description": "Needs more hand-holding, clearer error messages, more conceptual scaffolding."
|
||||
},
|
||||
{
|
||||
"label": "D) Let me describe my target developer",
|
||||
"description": "The three options above don't capture my primary user."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D1 \u2014 Who is your target developer?\n\nProject/branch: EvalKit SDK 2.0.0b1 beta release polish / main\n\nELI10: Different developers have radically different expectations for SDKs. An ML researcher wants speed to first eval. A platform engineer vets reliability and CI integration. A team lead evaluates upgrade safety. Getting this wrong means the whole review optimizes for the wrong person.\n\nStakes if we pick wrong: We polish the wrong touchpoints. A platform engineer's concerns (SLA docs, SBOM) get solved while an ML engineer's pain (5-min CI block, auth error, missing quickstart) stays broken.\n\nRecommendation: A because README explicitly identifies this persona and the onboarding study was run with them in mind.\n\nCompleteness: Note: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) ML engineer / Python daily driver (recommended)\n \u2714 Matches the README persona exactly: writes Python daily, uses terminal, wants local result before CI\n \u2714 Onboarding study was run with this persona; benchmark data is directly applicable\n \u274c May miss concerns of platform teams who deploy EvalKit in shared CI pipelines\nB) Platform / DevOps engineer\n \u2714 Covers reliability, SLA, and enterprise CI integration concerns that ML engineers may not raise\n \u274c Not the stated persona in README; polishing for them first would misalign the review\nC) Data scientist / less Python-fluent\n \u2714 Would catch gaps in error message clarity and conceptual scaffolding\n \u274c Over-indexes on hand-holding for an SDK whose README assumes daily Python usage\n\nNet: Confirming the persona anchors every friction-point recommendation to the person who will actually be doing the onboarding.\n\n<gstack-qid:plan-devex-review-persona>": "A) ML engineer / Python daily driver (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:00:16.505Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01SMNULqNnT5tKroY8b6kQm9",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D2 \u2014 Does this empathy narrative match reality?\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: Before scoring anything, I need to verify my mental model of what the developer actually experiences. A wrong model leads to wrong recommendations.\n\nStakes if we pick wrong: I polish the wrong path. If I assume they hit the demo command immediately but they actually try the quickstart first, every friction-point estimate is off.\n\n---\n\n**Developer Perspective: First 10 Minutes with EvalKit**\n\nI'm an ML engineer. My team evaluates LLM outputs manually today. EvalKit promises to automate this. I have 15 minutes before my next meeting.\n\nI open the README. The first heading is `# EvalKit SDK`. The pitch is clear. I scroll to Getting Started.\n\n`pip install evalkit==2.0.0b1` \u2014 fine, I run it. Sets EVALKIT_API_KEY \u2014 I go to my dashboard, copy my key, export it.\n\nThen: *\"follow the quickstart's command: `python examples/first_eval.py`\"*\n\nI run it. `python: can't open file 'examples/first_eval.py': [Errno 2] No such file or directory`\n\nI didn't expect that. I look at the README again. Ah \u2014 there's another command further down: `python -m evalkit.demo`. I run that instead.\n\nNothing happens. The cursor blinks. No output. No spinner. No \"Connecting to CI...\" message. I wait. I wait more. After 4 minutes I check if my terminal is frozen. After 5 minutes, results print. It worked. But I have no idea what just happened or why it took 5 minutes.\n\nTotal time: 7 minutes including the wrong path. I'm over my TTHW target by 3.5x. And I still don't know if this will be fast in CI or always take 5 minutes.\n\n---\n\nRecommendation: A \u2014 if this is accurate, these are the exact friction points the plan should address.\n\nCompleteness: Note: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) Accurate \u2014 proceed with this understanding (recommended)\n \u2714 All friction points identified are real; review recommendations will target the right problems\n \u2714 The missing quickstart file and the 5-minute silent wait are the two critical issues to fix\n \u274c None \u2014 if accurate, the path forward is clear\nB) Some of this is wrong \u2014 let me correct it\n \u2714 Correcting my model now prevents misaligned recommendations\n \u274c Requires you to describe the actual path, adding a round trip\nC) Way off \u2014 the actual experience is different\n \u2714 Ensures the whole review is grounded in real developer behavior\n \u274c Means I misread the docs and need to restart empathy modeling\n\nNet: The narrative is the lens for every friction-point question that follows. Getting it right now saves multiple wrong recommendations later.\n\n<gstack-qid:plan-devex-review-empathy>",
|
||||
"header": "Empathy narrative",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Accurate \u2014 proceed with this understanding (Recommended)",
|
||||
"description": "Missing quickstart file + 5-minute silent block are the two critical friction points; rest of the path matches."
|
||||
},
|
||||
{
|
||||
"label": "B) Some of this is wrong \u2014 let me correct it",
|
||||
"description": "Part of the narrative is off; I'll provide corrections before we proceed."
|
||||
},
|
||||
{
|
||||
"label": "C) Way off \u2014 the actual experience is different",
|
||||
"description": "The described path doesn't match reality; I'll explain what actually happens."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D2 \u2014 Does this empathy narrative match reality?\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: Before scoring anything, I need to verify my mental model of what the developer actually experiences. A wrong model leads to wrong recommendations.\n\nStakes if we pick wrong: I polish the wrong path. If I assume they hit the demo command immediately but they actually try the quickstart first, every friction-point estimate is off.\n\n---\n\n**Developer Perspective: First 10 Minutes with EvalKit**\n\nI'm an ML engineer. My team evaluates LLM outputs manually today. EvalKit promises to automate this. I have 15 minutes before my next meeting.\n\nI open the README. The first heading is `# EvalKit SDK`. The pitch is clear. I scroll to Getting Started.\n\n`pip install evalkit==2.0.0b1` \u2014 fine, I run it. Sets EVALKIT_API_KEY \u2014 I go to my dashboard, copy my key, export it.\n\nThen: *\"follow the quickstart's command: `python examples/first_eval.py`\"*\n\nI run it. `python: can't open file 'examples/first_eval.py': [Errno 2] No such file or directory`\n\nI didn't expect that. I look at the README again. Ah \u2014 there's another command further down: `python -m evalkit.demo`. I run that instead.\n\nNothing happens. The cursor blinks. No output. No spinner. No \"Connecting to CI...\" message. I wait. I wait more. After 4 minutes I check if my terminal is frozen. After 5 minutes, results print. It worked. But I have no idea what just happened or why it took 5 minutes.\n\nTotal time: 7 minutes including the wrong path. I'm over my TTHW target by 3.5x. And I still don't know if this will be fast in CI or always take 5 minutes.\n\n---\n\nRecommendation: A \u2014 if this is accurate, these are the exact friction points the plan should address.\n\nCompleteness: Note: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) Accurate \u2014 proceed with this understanding (recommended)\n \u2714 All friction points identified are real; review recommendations will target the right problems\n \u2714 The missing quickstart file and the 5-minute silent wait are the two critical issues to fix\n \u274c None \u2014 if accurate, the path forward is clear\nB) Some of this is wrong \u2014 let me correct it\n \u2714 Correcting my model now prevents misaligned recommendations\n \u274c Requires you to describe the actual path, adding a round trip\nC) Way off \u2014 the actual experience is different\n \u2714 Ensures the whole review is grounded in real developer behavior\n \u274c Means I misread the docs and need to restart empathy modeling\n\nNet: The narrative is the lens for every friction-point question that follows. Getting it right now saves multiple wrong recommendations later.\n\n<gstack-qid:plan-devex-review-empathy>": "A) Accurate \u2014 proceed with this understanding (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:00:44.593Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01LnujXYWVsRh1zAvuRYesXL",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D3 \u2014 Where do you want EvalKit to land on TTHW?\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: Time-to-Hello-World (TTHW) is how long a new developer takes from zero to seeing their first real result. Under 2 minutes gives 3-4x higher adoption than over 5 minutes. Right now EvalKit takes 6 minutes \u2014 the same 5-minute CI check that blocks production also blocks the onboarding demo. The agreed target is under 2 minutes, but the plan currently retains the CI block unchanged. These two facts are in direct conflict.\n\nStakes if we pick wrong: If we keep the CI block and the 2-minute target, both goals fail. If we acknowledge the tension now, the plan can propose a skip flag or fast-path that resolves it within the beta release scope.\n\n**COMPETITIVE DX BENCHMARK**\n```\nTool | TTHW | Notable DX Choice | Source\nPeer SDK A | 2 min | No mandatory remote check at T0 | docs/benchmarks.md\nPeer SDK C | 3 min | Offline mode available | docs/benchmarks.md\nPeer SDK B | 4 min | Requires account setup | docs/benchmarks.md\nEvalKit 2.0.0b1 | 6 min | Mandatory 5-min CI block on T0 | docs/current-contracts.md\nEvalKit target | <2 min | Agreed target per benchmarks.md | docs/benchmarks.md\n```\n\nThe mandatory CI block is the only reason EvalKit is 3x over its own target. Install + env var + demo command = ~1 minute. The CI block = +5 minutes.\n\nRecommendation: A \u2014 Champion tier is the stated target per benchmarks.md. The plan should add a `--skip-ci` flag or offline first-run path as a required fix, not a nice-to-have.\n\nCompleteness: A=9/10, B=7/10, C=5/10.\n\nPros / cons:\nA) Champion tier (<2 min) \u2014 add skip flag or offline path for first-run demo (recommended)\n \u2714 Matches the agreed target from benchmarks.md; closes the only gap between current and target\n \u2714 `python -m evalkit.demo` becomes a genuine magical moment at under 60 seconds\n \u274c Requires a plan change: add `--skip-ci` or offline-first-run path to the beta release scope\nB) Competitive tier (2\u20135 min) \u2014 add progress output to CI block, accept 6-min TTHW for now\n \u2714 Lower scope change: just add a spinner/progress indicator to the CI wait\n \u2714 Keeps CI gate intact without any architectural change\n \u274c Still 3x over the agreed target; falls behind Peer SDK A on the key metric the team already measured\nC) Current trajectory (6 min) \u2014 accept and update the target\n \u2714 Zero scope change; no new flags or paths needed\n \u274c Contradicts the agreed target; makes the completed onboarding study irrelevant\n\nNet: The CI block and the TTHW target are currently incompatible. The plan needs to either drop the target or add a fast-path for the demo. This is the single highest-leverage DX decision in the release.\n\n<gstack-qid:plan-devex-review-tthw-tier>",
|
||||
"header": "TTHW target",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Champion tier (<2 min) \u2014 add skip flag / offline path (Recommended)",
|
||||
"description": "Adds --skip-ci or offline-first-run path to beta scope. Closes the gap between current behavior and the agreed target."
|
||||
},
|
||||
{
|
||||
"label": "B) Competitive tier (2\u20135 min) \u2014 add progress output, accept 6-min TTHW",
|
||||
"description": "Adds a spinner/progress indicator to the CI block. Lower scope change but still 3x over the agreed target."
|
||||
},
|
||||
{
|
||||
"label": "C) Current trajectory (6 min) \u2014 accept and update the target",
|
||||
"description": "No scope change. Explicitly acknowledges the target won't be met this release."
|
||||
},
|
||||
{
|
||||
"label": "D) Tell me what's realistic for our constraints",
|
||||
"description": "Our CI architecture makes a skip flag harder than it looks; I'll explain."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D3 \u2014 Where do you want EvalKit to land on TTHW?\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: Time-to-Hello-World (TTHW) is how long a new developer takes from zero to seeing their first real result. Under 2 minutes gives 3-4x higher adoption than over 5 minutes. Right now EvalKit takes 6 minutes \u2014 the same 5-minute CI check that blocks production also blocks the onboarding demo. The agreed target is under 2 minutes, but the plan currently retains the CI block unchanged. These two facts are in direct conflict.\n\nStakes if we pick wrong: If we keep the CI block and the 2-minute target, both goals fail. If we acknowledge the tension now, the plan can propose a skip flag or fast-path that resolves it within the beta release scope.\n\n**COMPETITIVE DX BENCHMARK**\n```\nTool | TTHW | Notable DX Choice | Source\nPeer SDK A | 2 min | No mandatory remote check at T0 | docs/benchmarks.md\nPeer SDK C | 3 min | Offline mode available | docs/benchmarks.md\nPeer SDK B | 4 min | Requires account setup | docs/benchmarks.md\nEvalKit 2.0.0b1 | 6 min | Mandatory 5-min CI block on T0 | docs/current-contracts.md\nEvalKit target | <2 min | Agreed target per benchmarks.md | docs/benchmarks.md\n```\n\nThe mandatory CI block is the only reason EvalKit is 3x over its own target. Install + env var + demo command = ~1 minute. The CI block = +5 minutes.\n\nRecommendation: A \u2014 Champion tier is the stated target per benchmarks.md. The plan should add a `--skip-ci` flag or offline first-run path as a required fix, not a nice-to-have.\n\nCompleteness: A=9/10, B=7/10, C=5/10.\n\nPros / cons:\nA) Champion tier (<2 min) \u2014 add skip flag or offline path for first-run demo (recommended)\n \u2714 Matches the agreed target from benchmarks.md; closes the only gap between current and target\n \u2714 `python -m evalkit.demo` becomes a genuine magical moment at under 60 seconds\n \u274c Requires a plan change: add `--skip-ci` or offline-first-run path to the beta release scope\nB) Competitive tier (2\u20135 min) \u2014 add progress output to CI block, accept 6-min TTHW for now\n \u2714 Lower scope change: just add a spinner/progress indicator to the CI wait\n \u2714 Keeps CI gate intact without any architectural change\n \u274c Still 3x over the agreed target; falls behind Peer SDK A on the key metric the team already measured\nC) Current trajectory (6 min) \u2014 accept and update the target\n \u2714 Zero scope change; no new flags or paths needed\n \u274c Contradicts the agreed target; makes the completed onboarding study irrelevant\n\nNet: The CI block and the TTHW target are currently incompatible. The plan needs to either drop the target or add a fast-path for the demo. This is the single highest-leverage DX decision in the release.\n\n<gstack-qid:plan-devex-review-tthw-tier>": "A) Champion tier (<2 min) \u2014 add skip flag / offline path (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:01:14.715Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_017UnYmcondMyYq73XCB4WwU",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D4 \u2014 Confirm magical moment delivery vehicle\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The magical moment is when a developer stops asking 'is this worth my time?' and starts thinking 'I want to use this.' The delivery vehicle is HOW they experience that moment. README already chose one \u2014 I want to confirm it and note what changes with the champion-tier TTHW goal.\n\nStakes if we pick wrong: A demo command that silently blocks for 5 minutes is the opposite of magical. If we add the skip flag (from D3) and progress output, the same command becomes the magical moment.\n\nRecommendation: A \u2014 the copy-paste demo command is already the right choice. With the skip flag added (D3), it becomes magical instead of painful.\n\nNote: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) Keep `python -m evalkit.demo` \u2014 add skip flag + progress output (recommended)\n \u2714 Zero new infrastructure; the module already works and is in the published package\n \u2714 With skip flag + a 5-second spinner-then-results flow, the demo becomes genuinely magical for the ML engineer persona\n \u274c Still requires the plan to specify what the demo output looks like and what constitutes 'success'\nB) Add an interactive playground / browser-based sandbox\n \u2714 Zero install barrier; broadest reach across dev environments\n \u274c Out of scope for beta release per current-contracts.md (no new hosted service proposed)\n\nNet: The delivery vehicle is already chosen and right. The work is fixing the demo's behavior (skip flag, progress output, defined success output) so it actually delivers the magical moment.\n\n<gstack-qid:plan-devex-review-magical-moment>",
|
||||
"header": "Magical moment",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Keep python -m evalkit.demo \u2014 add skip flag + progress output (Recommended)",
|
||||
"description": "Keeps existing delivery vehicle. Plan adds: skip flag for CI block, progress indicator during wait, defined success output."
|
||||
},
|
||||
{
|
||||
"label": "B) Consider browser-based sandbox instead",
|
||||
"description": "Zero-install path, but out of scope for beta per current-contracts.md."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D4 \u2014 Confirm magical moment delivery vehicle\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The magical moment is when a developer stops asking 'is this worth my time?' and starts thinking 'I want to use this.' The delivery vehicle is HOW they experience that moment. README already chose one \u2014 I want to confirm it and note what changes with the champion-tier TTHW goal.\n\nStakes if we pick wrong: A demo command that silently blocks for 5 minutes is the opposite of magical. If we add the skip flag (from D3) and progress output, the same command becomes the magical moment.\n\nRecommendation: A \u2014 the copy-paste demo command is already the right choice. With the skip flag added (D3), it becomes magical instead of painful.\n\nNote: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) Keep `python -m evalkit.demo` \u2014 add skip flag + progress output (recommended)\n \u2714 Zero new infrastructure; the module already works and is in the published package\n \u2714 With skip flag + a 5-second spinner-then-results flow, the demo becomes genuinely magical for the ML engineer persona\n \u274c Still requires the plan to specify what the demo output looks like and what constitutes 'success'\nB) Add an interactive playground / browser-based sandbox\n \u2714 Zero install barrier; broadest reach across dev environments\n \u274c Out of scope for beta release per current-contracts.md (no new hosted service proposed)\n\nNet: The delivery vehicle is already chosen and right. The work is fixing the demo's behavior (skip flag, progress output, defined success output) so it actually delivers the magical moment.\n\n<gstack-qid:plan-devex-review-magical-moment>": "A) Keep python -m evalkit.demo \u2014 add skip flag + progress output (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:01:38.815Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01S2XA2rcBhQwaC4wdPamBkd",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D5 \u2014 Journey Stage: INSTALL\n\nFriction point: README documents a quickstart command that fails\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The README's Getting Started section tells developers to run `python examples/first_eval.py`. But docs/package-contents.txt explicitly states that file is 'absent from both the published package and the release examples archive.' Every developer who follows the documented path hits a FileNotFoundError before they see a single eval result. This is a broken front door.\n\nStakes if we pick wrong: Every first-time developer following the README fails. The demo command (`python -m evalkit.demo`) works, but developers only find it by reading further or by accident. This is the highest-probability abandonment point in the current flow.\n\nRecommendation: A \u2014 replace the broken reference. The working demo command is already in the README; it just needs to be the primary command.\n\nCompleteness: A=10/10, B=7/10, C=5/10.\n\nPros / cons:\nA) Fix: replace `python examples/first_eval.py` with `python -m evalkit.demo` as the primary getting-started command (recommended)\n \u2714 Closes the broken front door completely; developer follows README and immediately reaches the working path\n \u2714 Zero new code needed; evalkit.demo is already in the published package per package-contents.txt\n \u274c Removes the quickstart script reference, which may exist in external docs, blog posts, or prior comms\nB) Ship examples/first_eval.py in the package\n \u2714 Restores the documented contract without changing the README\n \u274c Requires creating and shipping a new file; that file also needs to handle the CI block, which circles back to D3\nC) Add an inline note: 'examples/first_eval.py is not in the published package, use `python -m evalkit.demo`'\n \u2714 Honest; doesn't change any code or package contents\n \u274c The README still starts with a broken command; developers who copy-paste and stop reading hit the error\n\nNet: The README documents a path that breaks on first run. This needs to be fixed before any other polishing matters.\n\n<gstack-qid:plan-devex-review-fp-quickstart>",
|
||||
"header": "Quickstart file",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Fix: replace broken reference with python -m evalkit.demo (Recommended)",
|
||||
"description": "Replace the primary getting-started command with the working demo module. Zero new code needed."
|
||||
},
|
||||
{
|
||||
"label": "B) Ship examples/first_eval.py in the package",
|
||||
"description": "Create and include the file. More scope, but preserves the documented contract."
|
||||
},
|
||||
{
|
||||
"label": "C) Add an inline note directing developers to the demo command",
|
||||
"description": "Honest but still starts with a broken command. Lower friction fix, lower coverage."
|
||||
},
|
||||
{
|
||||
"label": "D) Acceptable friction \u2014 skip",
|
||||
"description": "Leave the README as-is and accept that developers will work around it."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D5 \u2014 Journey Stage: INSTALL\n\nFriction point: README documents a quickstart command that fails\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The README's Getting Started section tells developers to run `python examples/first_eval.py`. But docs/package-contents.txt explicitly states that file is 'absent from both the published package and the release examples archive.' Every developer who follows the documented path hits a FileNotFoundError before they see a single eval result. This is a broken front door.\n\nStakes if we pick wrong: Every first-time developer following the README fails. The demo command (`python -m evalkit.demo`) works, but developers only find it by reading further or by accident. This is the highest-probability abandonment point in the current flow.\n\nRecommendation: A \u2014 replace the broken reference. The working demo command is already in the README; it just needs to be the primary command.\n\nCompleteness: A=10/10, B=7/10, C=5/10.\n\nPros / cons:\nA) Fix: replace `python examples/first_eval.py` with `python -m evalkit.demo` as the primary getting-started command (recommended)\n \u2714 Closes the broken front door completely; developer follows README and immediately reaches the working path\n \u2714 Zero new code needed; evalkit.demo is already in the published package per package-contents.txt\n \u274c Removes the quickstart script reference, which may exist in external docs, blog posts, or prior comms\nB) Ship examples/first_eval.py in the package\n \u2714 Restores the documented contract without changing the README\n \u274c Requires creating and shipping a new file; that file also needs to handle the CI block, which circles back to D3\nC) Add an inline note: 'examples/first_eval.py is not in the published package, use `python -m evalkit.demo`'\n \u2714 Honest; doesn't change any code or package contents\n \u274c The README still starts with a broken command; developers who copy-paste and stop reading hit the error\n\nNet: The README documents a path that breaks on first run. This needs to be fixed before any other polishing matters.\n\n<gstack-qid:plan-devex-review-fp-quickstart>": "A) Fix: replace broken reference with python -m evalkit.demo (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:02:00.898Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01NpxhVAQySBDEpJi3bGcvCa",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D6 \u2014 Journey Stage: HELLO WORLD\n\nFriction point: AuthError('request failed') gives no actionable information\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: When a developer has a bad or missing API key, the SDK currently raises `AuthError('request failed')`. That message tells them: something went wrong. It doesn't tell them: it was their API key, or how to fix it. An ML engineer seeing this error has to guess the cause. A good error message identifies the problem, explains the cause, and shows the fix in one shot.\n\nStakes if we pick wrong: A developer who fat-fingered their API key, or who set it as EVALKIT_KEY instead of EVALKIT_API_KEY, sees 'request failed' and has no idea what to do next. They file a support ticket or give up. The plan currently retains this message unchanged.\n\nRecommendation: A \u2014 fix the error message. This is exactly the kind of touchpoint DX POLISH exists to fix. It requires changing one string in the SDK, costs nothing, and eliminates a guaranteed support-ticket scenario.\n\nCompleteness: A=10/10, B=7/10, C=3/10.\n\nPros / cons:\nA) Fix: `AuthError('API key rejected. Check EVALKIT_API_KEY or generate a new key at evalkit.io/keys')` (recommended)\n \u2714 Identifies the problem (API key), explains the cause (rejected), shows the fix (check env var or regenerate key)\n \u2714 Follows the pattern already used by other SDK errors per docs/current-contracts.md ('identify cause, relevant argument, actionable fix')\n \u274c Requires updating the error message in the SDK codebase and in any error message docs\nB) Add a code field only: `AuthError(code='auth_invalid', message='request failed')`\n \u2714 Structured; allows API consumers to detect and handle auth errors programmatically\n \u274c The human-readable message is still useless; developer still doesn't know what to do\nC) Retain current: `AuthError('request failed')` per the plan\n \u2714 Zero change; preserves existing behavior\n \u274c The plan explicitly asks us to review this error; retaining it means the DX POLISH review found nothing fixable, which is false\n\nNet: The rest of the SDK's errors already identify cause, argument, and fix (per current-contracts.md). The auth error is the only one that doesn't. Fixing it brings it in line with the rest of the SDK's own standard.\n\n<gstack-qid:plan-devex-review-fp-autherror>",
|
||||
"header": "Auth error",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Fix: add cause + actionable fix to AuthError message (Recommended)",
|
||||
"description": "Change message to identify the API key as the problem and direct developers to check EVALKIT_API_KEY or regenerate."
|
||||
},
|
||||
{
|
||||
"label": "B) Add a structured error code only \u2014 keep existing prose message",
|
||||
"description": "Adds programmatic error code without fixing the human-readable message."
|
||||
},
|
||||
{
|
||||
"label": "C) Retain current AuthError('request failed') per the plan",
|
||||
"description": "Zero change. Accepts the message as-is."
|
||||
},
|
||||
{
|
||||
"label": "D) Acceptable friction \u2014 skip",
|
||||
"description": "The auth error is not a priority for this release."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D6 \u2014 Journey Stage: HELLO WORLD\n\nFriction point: AuthError('request failed') gives no actionable information\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: When a developer has a bad or missing API key, the SDK currently raises `AuthError('request failed')`. That message tells them: something went wrong. It doesn't tell them: it was their API key, or how to fix it. An ML engineer seeing this error has to guess the cause. A good error message identifies the problem, explains the cause, and shows the fix in one shot.\n\nStakes if we pick wrong: A developer who fat-fingered their API key, or who set it as EVALKIT_KEY instead of EVALKIT_API_KEY, sees 'request failed' and has no idea what to do next. They file a support ticket or give up. The plan currently retains this message unchanged.\n\nRecommendation: A \u2014 fix the error message. This is exactly the kind of touchpoint DX POLISH exists to fix. It requires changing one string in the SDK, costs nothing, and eliminates a guaranteed support-ticket scenario.\n\nCompleteness: A=10/10, B=7/10, C=3/10.\n\nPros / cons:\nA) Fix: `AuthError('API key rejected. Check EVALKIT_API_KEY or generate a new key at evalkit.io/keys')` (recommended)\n \u2714 Identifies the problem (API key), explains the cause (rejected), shows the fix (check env var or regenerate key)\n \u2714 Follows the pattern already used by other SDK errors per docs/current-contracts.md ('identify cause, relevant argument, actionable fix')\n \u274c Requires updating the error message in the SDK codebase and in any error message docs\nB) Add a code field only: `AuthError(code='auth_invalid', message='request failed')`\n \u2714 Structured; allows API consumers to detect and handle auth errors programmatically\n \u274c The human-readable message is still useless; developer still doesn't know what to do\nC) Retain current: `AuthError('request failed')` per the plan\n \u2714 Zero change; preserves existing behavior\n \u274c The plan explicitly asks us to review this error; retaining it means the DX POLISH review found nothing fixable, which is false\n\nNet: The rest of the SDK's errors already identify cause, argument, and fix (per current-contracts.md). The auth error is the only one that doesn't. Fixing it brings it in line with the rest of the SDK's own standard.\n\n<gstack-qid:plan-devex-review-fp-autherror>": "A) Fix: add cause + actionable fix to AuthError message (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:02:20.974Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01Du5Em5MKwPb3SwevTJsWsq",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D7 \u2014 Journey Stage: REAL USAGE\n\nFriction point: run_eval and run_batch take the same arguments in reversed order\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The two main evaluation functions take the same two arguments (dataset and evaluator) but in opposite order. `run_eval(dataset, evaluator)` vs `run_batch(evaluator, dataset)`. Any developer who learns one and uses the other will pass arguments in the wrong order, get wrong results with no error, and spend time debugging. No error is thrown because both arguments are the same type. This is the API equivalent of two buttons that look identical but do opposite things.\n\nStakes if we pick wrong: Developers using both functions will silently get wrong evaluation results. This is worse than an error \u2014 they won't know something went wrong until they examine the outputs carefully.\n\nRecommendation: A \u2014 standardize the argument order. The convention should be (dataset, evaluator) since run_eval already uses it and 'what you evaluate' logically comes before 'how to evaluate it'.\n\nCompleteness: A=10/10, B=7/10, C=3/10.\n\nPros / cons:\nA) Fix: standardize both to (dataset, evaluator) and add a deprecation warning if run_batch is called with wrong order (recommended)\n \u2714 Eliminates the silent-wrong-result trap; consistent argument order means muscle memory works across both functions\n \u2714 (dataset, evaluator) matches run_eval which came first; changing run_batch is the smaller surface area change\n \u274c Any existing code calling run_batch(evaluator, dataset) breaks silently; needs a migration note in the changelog\nB) Require keyword arguments for both functions\n \u2714 Makes argument order irrelevant; explicit is better than implicit\n \u274c Breaking change for all existing callers using positional args; higher migration burden than changing order\nC) Retain reversed order as documented\n \u2714 Zero change; no migration needed\n \u274c The plan explicitly asks us to review public function signatures; retaining a known cognitive trap is the opposite of DX POLISH\n\nNet: Two functions, same arguments, opposite order = a guaranteed debugging session for any developer who uses both. This is the kind of API inconsistency that drives 1-star SDK reviews.\n\n<gstack-qid:plan-devex-review-fp-argorder>",
|
||||
"header": "Arg order",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Fix: standardize both to (dataset, evaluator) (Recommended)",
|
||||
"description": "Standardize run_batch to (dataset, evaluator) to match run_eval. Add changelog note for existing run_batch callers."
|
||||
},
|
||||
{
|
||||
"label": "B) Require keyword arguments for both functions",
|
||||
"description": "Makes positional order irrelevant. Breaking change for all existing positional callers."
|
||||
},
|
||||
{
|
||||
"label": "C) Retain reversed order as documented",
|
||||
"description": "Zero change. Accepts the inconsistency as intentional."
|
||||
},
|
||||
{
|
||||
"label": "D) Acceptable friction \u2014 skip",
|
||||
"description": "The argument order is not a priority; most developers will use one function or the other, not both."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D7 \u2014 Journey Stage: REAL USAGE\n\nFriction point: run_eval and run_batch take the same arguments in reversed order\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The two main evaluation functions take the same two arguments (dataset and evaluator) but in opposite order. `run_eval(dataset, evaluator)` vs `run_batch(evaluator, dataset)`. Any developer who learns one and uses the other will pass arguments in the wrong order, get wrong results with no error, and spend time debugging. No error is thrown because both arguments are the same type. This is the API equivalent of two buttons that look identical but do opposite things.\n\nStakes if we pick wrong: Developers using both functions will silently get wrong evaluation results. This is worse than an error \u2014 they won't know something went wrong until they examine the outputs carefully.\n\nRecommendation: A \u2014 standardize the argument order. The convention should be (dataset, evaluator) since run_eval already uses it and 'what you evaluate' logically comes before 'how to evaluate it'.\n\nCompleteness: A=10/10, B=7/10, C=3/10.\n\nPros / cons:\nA) Fix: standardize both to (dataset, evaluator) and add a deprecation warning if run_batch is called with wrong order (recommended)\n \u2714 Eliminates the silent-wrong-result trap; consistent argument order means muscle memory works across both functions\n \u2714 (dataset, evaluator) matches run_eval which came first; changing run_batch is the smaller surface area change\n \u274c Any existing code calling run_batch(evaluator, dataset) breaks silently; needs a migration note in the changelog\nB) Require keyword arguments for both functions\n \u2714 Makes argument order irrelevant; explicit is better than implicit\n \u274c Breaking change for all existing callers using positional args; higher migration burden than changing order\nC) Retain reversed order as documented\n \u2714 Zero change; no migration needed\n \u274c The plan explicitly asks us to review public function signatures; retaining a known cognitive trap is the opposite of DX POLISH\n\nNet: Two functions, same arguments, opposite order = a guaranteed debugging session for any developer who uses both. This is the kind of API inconsistency that drives 1-star SDK reviews.\n\n<gstack-qid:plan-devex-review-fp-argorder>": "A) Fix: standardize both to (dataset, evaluator) (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:02:39.040Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01WRs9KGaWXamNABjHcbPS2S",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D8 \u2014 Journey Stage: UPGRADE\n\nFriction point: v1 Client.evaluate() removed with no migration path\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: When developers upgrade from v1 to v2, `Client.evaluate()` is gone and replaced by `Client.run()`. No deprecation warning, no compatibility alias, no migration guide, no codemod. Any v1 user who does `pip install --upgrade evalkit` gets a silent AttributeError on their next eval run \u2014 probably in production. This is called an 'upgrade cliff' and it's one of the most reliable ways to destroy developer trust in an SDK.\n\nStakes if we pick wrong: Every v1 user who upgrades without reading the full changelog (most of them) will have their production evals silently fail with `AttributeError: 'Client' object has no attribute 'evaluate'`. They will immediately downgrade, stop upgrading, or switch SDKs.\n\nRecommendation: A \u2014 add a deprecation shim. `Client.evaluate()` calls `Client.run()` and logs a DeprecationWarning. Costs almost nothing, eliminates the upgrade cliff.\n\nCompleteness: A=10/10, B=8/10, C=3/10.\n\nPros / cons:\nA) Fix: add Client.evaluate() compatibility alias that calls Client.run() with a DeprecationWarning (recommended)\n \u2714 Zero-risk upgrade path for v1 users; their code works immediately, the warning tells them to migrate\n \u2714 Three lines of code in client.py; changelog entry takes 2 sentences; migration guide takes one paragraph\n \u274c Keeps v1 API surface in v2 codebase; must be removed in v3 (planned obsolescence, not technical debt)\nB) Add a migration guide only (no alias)\n \u2714 Documents the change clearly; developers who read it can migrate in minutes\n \u274c Requires every v1 user to read the migration guide before upgrading; most won't; they still hit AttributeError\nC) Retain hard break with no migration path per the plan\n \u2714 Cleanest v2 codebase; no legacy surface area\n \u274c Every v1 user who upgrades will get an AttributeError in their production code; this is the plan the DX review was asked to improve\n\nNet: A three-line compatibility shim eliminates a production-breaking upgrade cliff for all v1 users. The cost of shipping it is near zero. The cost of not shipping it is measured in lost users and support tickets.\n\n<gstack-qid:plan-devex-review-fp-upgrade>",
|
||||
"header": "v1 upgrade",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Fix: add Client.evaluate() compat alias + DeprecationWarning (Recommended)",
|
||||
"description": "Three-line shim in client.py. v1 code works immediately; warning tells developers to migrate. Remove in v3."
|
||||
},
|
||||
{
|
||||
"label": "B) Add a migration guide only \u2014 no compatibility alias",
|
||||
"description": "Documents the change clearly. Developers who read it can migrate; most won't before hitting AttributeError."
|
||||
},
|
||||
{
|
||||
"label": "C) Retain hard break per the plan",
|
||||
"description": "Zero code change. Accepts that all v1 users will hit AttributeError on upgrade."
|
||||
},
|
||||
{
|
||||
"label": "D) Acceptable friction \u2014 skip",
|
||||
"description": "v1 usage is small enough that the upgrade cliff is not a significant concern for this release."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D8 \u2014 Journey Stage: UPGRADE\n\nFriction point: v1 Client.evaluate() removed with no migration path\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: When developers upgrade from v1 to v2, `Client.evaluate()` is gone and replaced by `Client.run()`. No deprecation warning, no compatibility alias, no migration guide, no codemod. Any v1 user who does `pip install --upgrade evalkit` gets a silent AttributeError on their next eval run \u2014 probably in production. This is called an 'upgrade cliff' and it's one of the most reliable ways to destroy developer trust in an SDK.\n\nStakes if we pick wrong: Every v1 user who upgrades without reading the full changelog (most of them) will have their production evals silently fail with `AttributeError: 'Client' object has no attribute 'evaluate'`. They will immediately downgrade, stop upgrading, or switch SDKs.\n\nRecommendation: A \u2014 add a deprecation shim. `Client.evaluate()` calls `Client.run()` and logs a DeprecationWarning. Costs almost nothing, eliminates the upgrade cliff.\n\nCompleteness: A=10/10, B=8/10, C=3/10.\n\nPros / cons:\nA) Fix: add Client.evaluate() compatibility alias that calls Client.run() with a DeprecationWarning (recommended)\n \u2714 Zero-risk upgrade path for v1 users; their code works immediately, the warning tells them to migrate\n \u2714 Three lines of code in client.py; changelog entry takes 2 sentences; migration guide takes one paragraph\n \u274c Keeps v1 API surface in v2 codebase; must be removed in v3 (planned obsolescence, not technical debt)\nB) Add a migration guide only (no alias)\n \u2714 Documents the change clearly; developers who read it can migrate in minutes\n \u274c Requires every v1 user to read the migration guide before upgrading; most won't; they still hit AttributeError\nC) Retain hard break with no migration path per the plan\n \u2714 Cleanest v2 codebase; no legacy surface area\n \u274c Every v1 user who upgrades will get an AttributeError in their production code; this is the plan the DX review was asked to improve\n\nNet: A three-line compatibility shim eliminates a production-breaking upgrade cliff for all v1 users. The cost of shipping it is near zero. The cost of not shipping it is measured in lost users and support tickets.\n\n<gstack-qid:plan-devex-review-fp-upgrade>": "A) Fix: add Client.evaluate() compat alias + DeprecationWarning (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:02:59.133Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01NgWVo5ThZTnVXd2apKQ7UM",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D9 \u2014 First-time developer confusion report: which issues to address?\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: I roleplayed as the ML engineer persona attempting EvalKit from README. Here's the confusion log. Five friction points surfaced; D5\u2013D8 already resolved four of them individually. The fifth (silent CI wait) was resolved via D3. Confirm all five belong in the plan.\n\n```\nFIRST-TIME DEVELOPER REPORT\n============================\nPersona: ML engineer, Python daily driver\nAttempting: EvalKit 2.0.0b1 getting started\n\nT+0:00 Install succeeds. EVALKIT_API_KEY set. README says\n 'python examples/first_eval.py'. I run it.\nT+0:10 FileNotFoundError. No such file. I reread README.\n Nowhere does it warn the file is missing.\nT+0:45 Found 'python -m evalkit.demo' lower in README.\n I run it. Cursor blinks. Nothing.\nT+1:00 Still nothing. Is it hung? No spinner, no output.\nT+3:00 Still nothing. I google 'evalkit demo hanging'.\nT+5:00 Results print. It worked. 5-minute wait not documented\n anywhere in the getting-started path.\nT+5:30 I try to write a real eval. I type run_eval(...) then\n decide to use run_batch instead. Wrong arg order.\n Wrong results. No error.\nT+8:00 I upgrade from v1. My production Client.evaluate() call\n throws AttributeError. I pin back to v1.\nTotal: 7+ min TTHW, 1 AttributeError in prod, 1 silent\n wrong result. 0 actionable errors shown.\n```\n\nRecommendation: A \u2014 all five issues are already agreed (D3\u2013D8); confirm and move to scoring.\n\nNote: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) Address all five issues as agreed in D3\u2013D8 (recommended)\n \u2714 All five decisions are already made; this confirms the plan scope is complete\n \u2714 Together they get TTHW from 7+ minutes to under 2 minutes and eliminate every zero-information error\n \u274c Adds scope to the beta release; each fix is small but requires SDK changes and changelog entries\nB) Let me adjust priorities \u2014 some of these are lower priority than others\n \u2714 Allows de-prioritizing fixes that conflict with release timeline or architecture constraints\n \u274c Means re-opening some of D3\u2013D8; the confusion log shows each issue is a real developer hit\n\nNet: These five issues together turn a 7-minute frustrating onboarding into a sub-2-minute magical moment. Every one is small. Together they are the release.\n\n<gstack-qid:plan-devex-review-confusion-report>",
|
||||
"header": "Confusion report",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Address all five issues as agreed in D3\u2013D8 (Recommended)",
|
||||
"description": "Confirm the plan scope includes all five fixes. Move to scoring passes."
|
||||
},
|
||||
{
|
||||
"label": "B) Let me adjust priorities \u2014 some issues are lower priority",
|
||||
"description": "Re-open prioritization. Some friction points may not fit the release timeline."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D9 \u2014 First-time developer confusion report: which issues to address?\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: I roleplayed as the ML engineer persona attempting EvalKit from README. Here's the confusion log. Five friction points surfaced; D5\u2013D8 already resolved four of them individually. The fifth (silent CI wait) was resolved via D3. Confirm all five belong in the plan.\n\n```\nFIRST-TIME DEVELOPER REPORT\n============================\nPersona: ML engineer, Python daily driver\nAttempting: EvalKit 2.0.0b1 getting started\n\nT+0:00 Install succeeds. EVALKIT_API_KEY set. README says\n 'python examples/first_eval.py'. I run it.\nT+0:10 FileNotFoundError. No such file. I reread README.\n Nowhere does it warn the file is missing.\nT+0:45 Found 'python -m evalkit.demo' lower in README.\n I run it. Cursor blinks. Nothing.\nT+1:00 Still nothing. Is it hung? No spinner, no output.\nT+3:00 Still nothing. I google 'evalkit demo hanging'.\nT+5:00 Results print. It worked. 5-minute wait not documented\n anywhere in the getting-started path.\nT+5:30 I try to write a real eval. I type run_eval(...) then\n decide to use run_batch instead. Wrong arg order.\n Wrong results. No error.\nT+8:00 I upgrade from v1. My production Client.evaluate() call\n throws AttributeError. I pin back to v1.\nTotal: 7+ min TTHW, 1 AttributeError in prod, 1 silent\n wrong result. 0 actionable errors shown.\n```\n\nRecommendation: A \u2014 all five issues are already agreed (D3\u2013D8); confirm and move to scoring.\n\nNote: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) Address all five issues as agreed in D3\u2013D8 (recommended)\n \u2714 All five decisions are already made; this confirms the plan scope is complete\n \u2714 Together they get TTHW from 7+ minutes to under 2 minutes and eliminate every zero-information error\n \u274c Adds scope to the beta release; each fix is small but requires SDK changes and changelog entries\nB) Let me adjust priorities \u2014 some of these are lower priority than others\n \u2714 Allows de-prioritizing fixes that conflict with release timeline or architecture constraints\n \u274c Means re-opening some of D3\u2013D8; the confusion log shows each issue is a real developer hit\n\nNet: These five issues together turn a 7-minute frustrating onboarding into a sub-2-minute magical moment. Every one is small. Together they are the release.\n\n<gstack-qid:plan-devex-review-confusion-report>": "A) Address all five issues as agreed in D3\u2013D8 (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:03:29.271Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01UCDASxmA9i9j2g6zWjw4hs",
|
||||
"questions": [
|
||||
{
|
||||
"question": "gstack can search learnings from your other projects on this machine to find patterns that might apply here. This stays local (no data leaves your machine). Recommended for solo developers. Skip if you work on multiple client codebases where cross-contamination would be a concern.\n\n<gstack-qid:cross-project-learnings>",
|
||||
"header": "Cross-project",
|
||||
"options": [
|
||||
{
|
||||
"label": "Enable cross-project learnings (Recommended)",
|
||||
"description": "Search for patterns from other projects on this machine when running reviews."
|
||||
},
|
||||
{
|
||||
"label": "Keep learnings project-scoped only",
|
||||
"description": "Limit learnings search to this project only."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"gstack can search learnings from your other projects on this machine to find patterns that might apply here. This stays local (no data leaves your machine). Recommended for solo developers. Skip if you work on multiple client codebases where cross-contamination would be a concern.\n\n<gstack-qid:cross-project-learnings>": "Enable cross-project learnings (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:04:05.425Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01KeyeMLYxU3SHBPLgFuu8x1",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D10 \u2014 Pass 4: Documentation\n\nFinding: demo output is unspecified \u2014 developer has no reference point for 'success'\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The README says to run `python -m evalkit.demo` but shows no example output. After a 5-minute wait (or after adding the skip flag), the developer sees... something. But they don't know if they should see 10 scores, 100 scores, a single number, or a JSON blob. Without a reference, 'did this work?' requires guesswork. Stripe's docs show you the exact JSON you'll receive. EvalKit's README shows you nothing.\n\nStakes if we pick wrong: A developer who gets output but doesn't recognize it as success may assume it failed and re-run, open an issue, or abandon. This is especially bad for a 5-minute wait \u2014 after sitting through the CI block, they need instant confirmation that it worked.\n\nRecommendation: A \u2014 add expected output to the README. This is a one-paragraph doc change, zero code required, and it makes the first success moment unambiguous.\n\nCompleteness: A=10/10, B=7/10, C=3/10.\n\nPros / cons:\nA) Add to plan: include sample demo output in README under 'Getting Started' (recommended)\n \u2714 Developer immediately knows what success looks like; 'it worked!' moment is unambiguous\n \u2714 Zero code change; one paragraph in README; takes 10 minutes to write\n \u274c Sample output may drift if demo data changes (low risk: sample_responses.json is bundled and stable)\nB) Add as a TODO for post-beta docs polish\n \u2714 Defers the work without blocking the beta\n \u274c The demo is THE magical moment for this release; leaving its output undocumented weakens the whole TTHW fix\nC) Skip \u2014 developer will recognize success when they see scores\n \u2714 Zero effort\n \u274c ML engineers expect scores to be domain-specific; without a reference, 'are these scores correct?' is unanswerable\n\nNet: The getting-started flow now ends with a demo run. If the output isn't documented, the developer's first success moment is ambiguous. A 10-second read of sample output turns ambiguity into confidence.\n\n<gstack-qid:plan-devex-review-todo-demo-output>",
|
||||
"header": "Demo output doc",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Add to plan: include sample demo output in README (Recommended)",
|
||||
"description": "Show what success looks like. One paragraph, zero code change, takes 10 minutes."
|
||||
},
|
||||
{
|
||||
"label": "B) Add as a TODO for post-beta docs polish",
|
||||
"description": "Defer; not blocking the beta, but weakens the TTHW fix."
|
||||
},
|
||||
{
|
||||
"label": "C) Skip \u2014 developer will recognize success when they see scores",
|
||||
"description": "Zero effort. Accepts ambiguous success moment."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D10 \u2014 Pass 4: Documentation\n\nFinding: demo output is unspecified \u2014 developer has no reference point for 'success'\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The README says to run `python -m evalkit.demo` but shows no example output. After a 5-minute wait (or after adding the skip flag), the developer sees... something. But they don't know if they should see 10 scores, 100 scores, a single number, or a JSON blob. Without a reference, 'did this work?' requires guesswork. Stripe's docs show you the exact JSON you'll receive. EvalKit's README shows you nothing.\n\nStakes if we pick wrong: A developer who gets output but doesn't recognize it as success may assume it failed and re-run, open an issue, or abandon. This is especially bad for a 5-minute wait \u2014 after sitting through the CI block, they need instant confirmation that it worked.\n\nRecommendation: A \u2014 add expected output to the README. This is a one-paragraph doc change, zero code required, and it makes the first success moment unambiguous.\n\nCompleteness: A=10/10, B=7/10, C=3/10.\n\nPros / cons:\nA) Add to plan: include sample demo output in README under 'Getting Started' (recommended)\n \u2714 Developer immediately knows what success looks like; 'it worked!' moment is unambiguous\n \u2714 Zero code change; one paragraph in README; takes 10 minutes to write\n \u274c Sample output may drift if demo data changes (low risk: sample_responses.json is bundled and stable)\nB) Add as a TODO for post-beta docs polish\n \u2714 Defers the work without blocking the beta\n \u274c The demo is THE magical moment for this release; leaving its output undocumented weakens the whole TTHW fix\nC) Skip \u2014 developer will recognize success when they see scores\n \u2714 Zero effort\n \u274c ML engineers expect scores to be domain-specific; without a reference, 'are these scores correct?' is unanswerable\n\nNet: The getting-started flow now ends with a demo run. If the output isn't documented, the developer's first success moment is ambiguous. A 10-second read of sample output turns ambiguity into confidence.\n\n<gstack-qid:plan-devex-review-todo-demo-output>": "A) Add to plan: include sample demo output in README (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:05:45.958Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_011NdN2KRZrdiubotAmj4Ehk",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D11 \u2014 What next?\n\nProject/branch: EvalKit SDK 2.0.0b1 DX review complete / main\n\nELI10: The DX review is done. Eight tasks identified, all with clear files and verify steps. The --skip-ci flag and run_batch arg-order fix both touch client.py and have breaking-change implications \u2014 an eng review should validate both before implementation starts.\n\nRecommendation: A \u2014 run /plan-eng-review next. DX review found two API-level changes (T5 arg order, T1 skip-ci flag) that need architecture validation.\n\nNote: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) Run /plan-eng-review next (recommended)\n \u2714 Validates the --skip-ci flag design and the run_batch arg-order change before anyone starts coding\n \u2714 Required gate for shipping; DX review surfaced issues with architecture implications\n \u274c Adds ~20 minutes before implementation begins\nB) Ready to implement \u2014 run /devex-review after shipping\n \u2714 Start coding immediately with the 8 tasks as the spec\n \u274c Skips the required eng review gate; architectural risks in T1 and T5 are unvalidated\nC) Skip \u2014 I'll handle next steps manually\n \u2714 Full control over what happens next\n \u274c No gate between DX findings and implementation\n\nNet: Eight P1 tasks ready. T1 and T5 have breaking-change implications that warrant a quick eng review before implementation.\n\n<gstack-qid:plan-devex-review-next-steps>",
|
||||
"header": "Next steps",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Run /plan-eng-review next (Recommended)",
|
||||
"description": "Validates --skip-ci flag design and run_batch arg-order breaking change before coding starts."
|
||||
},
|
||||
{
|
||||
"label": "B) Ready to implement \u2014 run /devex-review after shipping",
|
||||
"description": "Start coding from the 8-task spec. Boomerang with /devex-review to verify TTHW target was met."
|
||||
},
|
||||
{
|
||||
"label": "C) Skip \u2014 I'll handle next steps manually",
|
||||
"description": "Full control over what happens next."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D11 \u2014 What next?\n\nProject/branch: EvalKit SDK 2.0.0b1 DX review complete / main\n\nELI10: The DX review is done. Eight tasks identified, all with clear files and verify steps. The --skip-ci flag and run_batch arg-order fix both touch client.py and have breaking-change implications \u2014 an eng review should validate both before implementation starts.\n\nRecommendation: A \u2014 run /plan-eng-review next. DX review found two API-level changes (T5 arg order, T1 skip-ci flag) that need architecture validation.\n\nNote: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) Run /plan-eng-review next (recommended)\n \u2714 Validates the --skip-ci flag design and the run_batch arg-order change before anyone starts coding\n \u2714 Required gate for shipping; DX review surfaced issues with architecture implications\n \u274c Adds ~20 minutes before implementation begins\nB) Ready to implement \u2014 run /devex-review after shipping\n \u2714 Start coding immediately with the 8 tasks as the spec\n \u274c Skips the required eng review gate; architectural risks in T1 and T5 are unvalidated\nC) Skip \u2014 I'll handle next steps manually\n \u2714 Full control over what happens next\n \u274c No gate between DX findings and implementation\n\nNet: Eight P1 tasks ready. T1 and T5 have breaking-change implications that warrant a quick eng review before implementation.\n\n<gstack-qid:plan-devex-review-next-steps>": "A) Run /plan-eng-review next (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:10:08.919Z"
|
||||
}
|
||||
],
|
||||
"assistantMessages": [
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "I see the preamble completed. Let me run the initial reconnaissance steps in parallel.",
|
||||
"timestamp": "2026-09-09T01:57:04.802Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "Good \u2014 I have a clear picture of all the artifacts. Let me check for a design doc and run context recovery.",
|
||||
"timestamp": "2026-09-09T01:58:58.132Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "I have all the context I need. No design doc exists, but the README + docs/benchmarks.md + docs/current-contracts.md + docs/api.md together give dense, structured context \u2014 better than many design docs I see.\n\nThe plan is clearly classified as a **Library/SDK** (Python SDK for ML engineers evaluating LLM responses), with mode pre-selected as **DX POLISH**, persona pre-defined, and magical moment delivery pre-chosen (`python -m evalkit.demo`).\n\nLet me offer the prereq skill before proceeding.",
|
||||
"timestamp": "2026-09-09T01:59:43.787Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "Proceeding with existing docs. Now confirming the developer persona (Step 0A) \u2014 the README pre-defines it, so I'll confirm rather than starting from scratch.",
|
||||
"timestamp": "2026-09-09T02:00:00.965Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "Good. Persona confirmed: ML engineer, Python daily driver. Now writing the empathy narrative (Step 0B) based on the actual README path.",
|
||||
"timestamp": "2026-09-09T02:00:25.370Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "Narrative confirmed. Now Step 0C \u2014 competitive benchmarking. I have benchmark data from docs/benchmarks.md.",
|
||||
"timestamp": "2026-09-09T02:00:52.148Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "TTHW: Champion tier confirmed. Now confirming the magical moment delivery (Step 0D) \u2014 already settled in README.",
|
||||
"timestamp": "2026-09-09T02:01:25.503Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "Mode is pre-selected in PLAN.md as DX POLISH. Confirming and moving to the journey trace.\n\nThe mode is DX POLISH \u2014 confirmed by PLAN.md. Now Step 0F: journey trace friction points. I'll fire one per AskUserQuestion.\n\n**Friction point 1 of 5 \u2014 Missing quickstart file**",
|
||||
"timestamp": "2026-09-09T02:01:44.591Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "**Friction point 2 of 5 \u2014 auth error message**",
|
||||
"timestamp": "2026-09-09T02:02:03.755Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "**Friction point 3 of 5 \u2014 reversed argument order in API**",
|
||||
"timestamp": "2026-09-09T02:02:23.239Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "**Friction point 4 of 5 \u2014 v1\u2192v2 upgrade cliff**",
|
||||
"timestamp": "2026-09-09T02:02:41.501Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "All 5 friction points addressed. Now Step 0G \u2014 first-time developer roleplay confusion report to confirm priorities.",
|
||||
"timestamp": "2026-09-09T02:03:07.099Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "Step 0 complete. Now reading the review sections to run the 8 DX passes.",
|
||||
"timestamp": "2026-09-09T02:03:33.011Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "Now running infrastructure checks in parallel before the 8 DX passes.",
|
||||
"timestamp": "2026-09-09T02:03:55.001Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "Codex review skipped (codex_reviews disabled). Re-enable: `gstack-config set codex_reviews enabled`",
|
||||
"timestamp": "2026-09-09T02:04:25.287Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "All prerequisite checks complete. Now running the 8 DX passes formally. No new issues in passes 1\u20133 beyond those resolved in D3\u2013D9. A new finding emerged in Pass 4.\n\n**Pass 1 \u2014 Getting Started: 2/10 \u2192 9/10 (after D3 + D5 fixes)**\n\nEvidence: 7-min TTHW vs 2-min target (benchmarks.md). Broken quickstart file (package-contents.txt). Silent 5-min CI block (current-contracts.md). Fixes: `--skip-ci` flag (D3), replace `examples/first_eval.py` with `python -m evalkit.demo` as primary command (D5).\n\n**Pass 2 \u2014 API/CLI/SDK: 3/10 \u2192 8/10 (after D7 fix)**\n\nEvidence: `run_eval(dataset, evaluator)` vs `run_batch(evaluator, dataset)` \u2014 same concepts, reversed order, no type enforcement, silent wrong results. Fix: standardize both to `(dataset, evaluator)` (D7).\n\n**Pass 3 \u2014 Error Messages: 5/10 \u2192 9/10 (after D6 fix)**\n\nEvidence: `AuthError(\"request failed\")` violates the SDK's own standard (current-contracts.md says other errors identify cause + argument + fix). Fix: add cause + actionable fix to auth error (D6).\n\n**Pass 4 \u2014 Documentation: 5/10 \u2192 8/10 (after D3/D5/D8 fixes)**\n\nEvidence: README references broken file (D5), no v1\u2192v2 migration path (D8), no documentation of the CI wait in the getting-started flow. New finding below.\n\n**Pass 4 has one new finding not covered by D3\u2013D9:**",
|
||||
"timestamp": "2026-09-09T02:05:27.888Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "Passes 5\u20138 complete. No new issues (D8 covers upgrade path; dev environment, community, and measurement are already solid per current-contracts.md/benchmarks.md). Now composing the full plan file with required outputs and review report.",
|
||||
"timestamp": "2026-09-09T02:06:28.408Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "Now writing to the task-specified path:",
|
||||
"timestamp": "2026-09-09T02:07:47.902Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "Now running the review log, JSONL tasks artifact, and readiness dashboard in parallel.",
|
||||
"timestamp": "2026-09-09T02:09:21.532Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "**Review Readiness Dashboard:**\n\n```\n+====================================================================+\n| REVIEW READINESS DASHBOARD |\n+====================================================================+\n| Review | Runs | Last Run | Status | Required |\n|-----------------|------|---------------------|-----------|----------|\n| Eng Review | 0 | \u2014 | \u2014 | YES |\n| CEO Review | 0 | \u2014 | \u2014 | no |\n| Design Review | 0 | \u2014 | \u2014 | no |\n| Adversarial | 0 | \u2014 | \u2014 | no |\n| Outside Voice | 1 | 2026-09-09 02:04 | disabled | no |\n| DX Review | 1 | 2026-09-09 02:09 | issues_found | \u2014 |\n+--------------------------------------------------------------------+\n| VERDICT: NOT CLEARED \u2014 Eng Review required before shipping |\n+====================================================================+\n```\n\nThe DX issues found have architectural implications: the `--skip-ci` flag touches both `evalkit/demo.py` and `evalkit/client.py`, and the `run_batch` arg-order change is a breaking change for existing callers. Eng review should validate both.",
|
||||
"timestamp": "2026-09-09T02:09:55.649Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "Now verifying the EXIT PLAN MODE GATE before calling ExitPlanMode.",
|
||||
"timestamp": "2026-09-09T02:10:39.592Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"text": "Gate check passes:\n- Last `##` heading is `## GSTACK REVIEW REPORT` \u2713\n- Table with Runs/Status/Findings present \u2713\n- VERDICT line present \u2713\n- Final non-whitespace line is exactly `NO UNRESOLVED DECISIONS` (unbolded) \u2713\n- `gstack-review-log` called \u2713\n- `gstack-review-read` run \u2713",
|
||||
"timestamp": "2026-09-09T02:10:51.897Z"
|
||||
}
|
||||
],
|
||||
"planReadyRequests": [
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01Qxmmvf72nJhsPSkAZip4EX",
|
||||
"timestamp": "2026-09-09T02:10:54.048Z",
|
||||
"failed": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"report": "# DX Review: EvalKit SDK 2.0.0b1 Beta Release Polish\n\n## Context\n\nEvalKit is a Python SDK for ML engineers evaluating LLM responses. This review\napplies **DX POLISH** mode to the existing release scope \u2014 no new surfaces, no new\nhosted services, no new language bindings. The goal is to make every documented\ntouchpoint bulletproof before the beta ships.\n\nFive developer-facing contracts in the current plan are actively harmful to the ML\nengineer persona: a broken quickstart reference, a silent 5-minute CI block, a\nzero-information auth error, reversed argument order between the two main functions,\nand a production-breaking v1\u2192v2 upgrade cliff. All five were confirmed by tracing\nthe actual getting-started path through README.md, docs/current-contracts.md,\ndocs/api.md, and docs/package-contents.txt.\n\n---\n\n## Developer Persona Card\n\n```\nTARGET DEVELOPER PERSONA\n========================\nWho: ML engineer \u2014 writes Python daily, uses a terminal\nContext: Evaluating whether EvalKit can replace manual LLM output review\n before integrating it into production CI\nTolerance: ~15 minutes before abandoning; will copy-paste README commands verbatim\nExpects: pip install + one command = working eval result; no guesswork,\n no waiting without feedback, no broken links\n```\n\n---\n\n## Developer Empathy Narrative\n\nI'm an ML engineer. My team evaluates LLM outputs manually today. EvalKit promises\nto automate this. I have 15 minutes before my next meeting.\n\nI open the README. The first heading is `# EvalKit SDK`. The pitch is clear. I scroll\nto Getting Started.\n\n`pip install evalkit==2.0.0b1` \u2014 fine, I run it. Sets EVALKIT_API_KEY \u2014 I go to my\ndashboard, copy my key, export it.\n\nThen: *\"follow the quickstart's command: `python examples/first_eval.py`\"*\n\nI run it. `python: can't open file 'examples/first_eval.py': [Errno 2] No such file or\ndirectory`\n\nI didn't expect that. I look at the README again. Ah \u2014 there's another command further\ndown: `python -m evalkit.demo`. I run that instead.\n\nNothing happens. The cursor blinks. No output. No spinner. No \"Connecting to CI...\"\nmessage. I wait. I wait more. After 4 minutes I check if my terminal is frozen. After\n5 minutes, results print. It worked. But I have no idea what just happened or why it\ntook 5 minutes \u2014 or whether what I see is the right output.\n\nTotal time: 7 minutes including the wrong path. I'm over my TTHW target by 3.5x. And I\nstill don't know if this will be fast in CI or always take 5 minutes.\n\n---\n\n## Competitive DX Benchmark\n\n```\nTool | TTHW | Notable DX Choice | Source\nPeer SDK A | 2 min | No mandatory remote check at T0 | docs/benchmarks.md\nPeer SDK C | 3 min | Offline mode available | docs/benchmarks.md\nPeer SDK B | 4 min | Requires account setup | docs/benchmarks.md\nEvalKit 2.0.0b1 | 6 min | Mandatory 5-min CI block on T0 | docs/current-contracts.md\nEvalKit target | <2 min | Agreed per benchmarks.md | docs/benchmarks.md\n```\n\nAfter fixes: EvalKit lands at **Champion tier (<2 min)** \u2014 matching the agreed target\nand beating all three peer SDKs.\n\n---\n\n## Magical Moment Specification\n\n**Delivery vehicle:** `python -m evalkit.demo` (copy-paste demo command)\n\n**Required implementation:**\n- Add `--skip-ci` flag (or `EVALKIT_SKIP_CI=1` env var) to `evalkit/demo.py`\n- Add progress output during CI wait: `\"Waiting for CI check... (up to 5 min)\"`\n with a spinner so developer knows the process is alive\n- Demo must complete in under 60 seconds with skip flag enabled\n- README must show expected demo output (sample per-example scores + overall score)\n so developer can immediately confirm success\n\n**Success state:**\n```\n$ python -m evalkit.demo --skip-ci\nEvaluating 10 sample responses...\n Response 1: 0.82\n Response 2: 0.76\n ...\n Response 10: 0.91\n\nOverall score: 0.84\nDemo complete. Run `evalkit --help` to evaluate your own data.\n```\n\n---\n\n## Developer Journey Map\n\n```\nSTAGE | DEVELOPER DOES | FRICTION POINTS | STATUS\n----------------|-----------------------------|-----------------------|--------\n1. Discover | Opens README, reads pitch | None | OK\n2. Install | pip install evalkit==2.0.0b1| None after README fix | FIXED\n3. Hello World | python -m evalkit.demo | Broken quickstart ref | FIXED (D5)\n | | Silent 5-min CI block | FIXED (D3)\n | | No reference output | FIXED (D10)\n4. Real Usage | run_eval / run_batch | Reversed arg order | FIXED (D7)\n5. Debug | Invalid API key | AuthError useless | FIXED (D6)\n6. Upgrade | pip install --upgrade | Hard v1\u2192v2 break | FIXED (D8)\n```\n\n---\n\n## FirstLine truncated
|
||||
"reportWrite": {
|
||||
"toolUseId": "toolu_01TALwv9dfC6EE8bc6gfzKo4",
|
||||
"requestedAt": "2026-09-09T02:09:18.107Z",
|
||||
"resultAt": "2026-09-09T02:09:19.183Z",
|
||||
"failed": false
|
||||
},
|
||||
"limits": [
|
||||
"Free replay dates the copied report from successful native Write completion; original filesystem mtime is not reconstructed.",
|
||||
"Raw O9setup4review and cancellation remain unchanged. All13 complete native calls are retained.",
|
||||
"Free replay start bound precedes this retry's first native question; it is not a claimed exact CLI start timestamp."
|
||||
],
|
||||
"startedAt": "2026-09-09T01:56:15.000Z"
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"sourceObservation": ".context/ship-source-o-full-paid-20260909-0145/devex-retry-terminal-watch-v1/observation.json",
|
||||
"call": {
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01KeyeMLYxU3SHBPLgFuu8x1",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D10 \u2014 Pass 4: Documentation\n\nFinding: demo output is unspecified \u2014 developer has no reference point for 'success'\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The README says to run `python -m evalkit.demo` but shows no example output. After a 5-minute wait (or after adding the skip flag), the developer sees... something. But they don't know if they should see 10 scores, 100 scores, a single number, or a JSON blob. Without a reference, 'did this work?' requires guesswork. Stripe's docs show you the exact JSON you'll receive. EvalKit's README shows you nothing.\n\nStakes if we pick wrong: A developer who gets output but doesn't recognize it as success may assume it failed and re-run, open an issue, or abandon. This is especially bad for a 5-minute wait \u2014 after sitting through the CI block, they need instant confirmation that it worked.\n\nRecommendation: A \u2014 add expected output to the README. This is a one-paragraph doc change, zero code required, and it makes the first success moment unambiguous.\n\nCompleteness: A=10/10, B=7/10, C=3/10.\n\nPros / cons:\nA) Add to plan: include sample demo output in README under 'Getting Started' (recommended)\n \u2714 Developer immediately knows what success looks like; 'it worked!' moment is unambiguous\n \u2714 Zero code change; one paragraph in README; takes 10 minutes to write\n \u274c Sample output may drift if demo data changes (low risk: sample_responses.json is bundled and stable)\nB) Add as a TODO for post-beta docs polish\n \u2714 Defers the work without blocking the beta\n \u274c The demo is THE magical moment for this release; leaving its output undocumented weakens the whole TTHW fix\nC) Skip \u2014 developer will recognize success when they see scores\n \u2714 Zero effort\n \u274c ML engineers expect scores to be domain-specific; without a reference, 'are these scores correct?' is unanswerable\n\nNet: The getting-started flow now ends with a demo run. If the output isn't documented, the developer's first success moment is ambiguous. A 10-second read of sample output turns ambiguity into confidence.\n\n<gstack-qid:plan-devex-review-todo-demo-output>",
|
||||
"header": "Demo output doc",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Add to plan: include sample demo output in README (Recommended)",
|
||||
"description": "Show what success looks like. One paragraph, zero code change, takes 10 minutes."
|
||||
},
|
||||
{
|
||||
"label": "B) Add as a TODO for post-beta docs polish",
|
||||
"description": "Defer; not blocking the beta, but weakens the TTHW fix."
|
||||
},
|
||||
{
|
||||
"label": "C) Skip \u2014 developer will recognize success when they see scores",
|
||||
"description": "Zero effort. Accepts ambiguous success moment."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D10 \u2014 Pass 4: Documentation\n\nFinding: demo output is unspecified \u2014 developer has no reference point for 'success'\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The README says to run `python -m evalkit.demo` but shows no example output. After a 5-minute wait (or after adding the skip flag), the developer sees... something. But they don't know if they should see 10 scores, 100 scores, a single number, or a JSON blob. Without a reference, 'did this work?' requires guesswork. Stripe's docs show you the exact JSON you'll receive. EvalKit's README shows you nothing.\n\nStakes if we pick wrong: A developer who gets output but doesn't recognize it as success may assume it failed and re-run, open an issue, or abandon. This is especially bad for a 5-minute wait \u2014 after sitting through the CI block, they need instant confirmation that it worked.\n\nRecommendation: A \u2014 add expected output to the README. This is a one-paragraph doc change, zero code required, and it makes the first success moment unambiguous.\n\nCompleteness: A=10/10, B=7/10, C=3/10.\n\nPros / cons:\nA) Add to plan: include sample demo output in README under 'Getting Started' (recommended)\n \u2714 Developer immediately knows what success looks like; 'it worked!' moment is unambiguous\n \u2714 Zero code change; one paragraph in README; takes 10 minutes to write\n \u274c Sample output may drift if demo data changes (low risk: sample_responses.json is bundled and stable)\nB) Add as a TODO for post-beta docs polish\n \u2714 Defers the work without blocking the beta\n \u274c The demo is THE magical moment for this release; leaving its output undocumented weakens the whole TTHW fix\nC) Skip \u2014 developer will recognize success when they see scores\n \u2714 Zero effort\n \u274c ML engineers expect scores to be domain-specific; without a reference, 'are these scores correct?' is unanswerable\n\nNet: The getting-started flow now ends with a demo run. If the output isn't documented, the developer's first success moment is ambiguous. A 10-second read of sample output turns ambiguity into confidence.\n\n<gstack-qid:plan-devex-review-todo-demo-output>": "A) Add to plan: include sample demo output in README (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:05:45.958Z"
|
||||
}
|
||||
}
|
||||
+382
@@ -0,0 +1,382 @@
|
||||
{
|
||||
"calls": [
|
||||
{
|
||||
"id": "toolu_01SG2Hk1e1bh7J2NiZZZuFv2",
|
||||
"questions": [
|
||||
{
|
||||
"question": "Should gstack add skill routing rules to this project's CLAUDE.md?",
|
||||
"header": "Routing rules",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Skip — fixture repo (Recommended)",
|
||||
"description": "This is a plan-review fixture repo; routing rules in CLAUDE.md would be noise here. Skip."
|
||||
},
|
||||
{
|
||||
"label": "Add routing rules",
|
||||
"description": "Append the standard gstack skill routing section to CLAUDE.md and commit it."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"question": "gstack can search learnings from your other projects on this machine to find patterns that apply here. This stays local — no data leaves your machine. Enable cross-project learnings?",
|
||||
"header": "Cross-proj learn",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Enable cross-project (Recommended)",
|
||||
"description": "Search learnings from all local projects. Best for solo developers. Recommended for most setups."
|
||||
},
|
||||
{
|
||||
"label": "Keep project-scoped only",
|
||||
"description": "Learnings only from this project. Safer if you work across multiple client codebases."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"Should gstack add skill routing rules to this project's CLAUDE.md?": "Skip — fixture repo (Recommended)",
|
||||
"gstack can search learnings from your other projects on this machine to find patterns that apply here. This stays local — no data leaves your machine. Enable cross-project learnings?": "Enable cross-project (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"timestamp": "2026-09-09T01:52:08.251Z",
|
||||
"answeredAt": "2026-09-09T01:52:17.131Z",
|
||||
"toolUseId": "toolu_01SG2Hk1e1bh7J2NiZZZuFv2",
|
||||
"sessionId": "c1849829-c0f7-42bc-b198-334a9cf2be9a"
|
||||
},
|
||||
{
|
||||
"id": "toolu_01TqHSgk5ec9XasVdGpYxVQ2",
|
||||
"questions": [
|
||||
{
|
||||
"question": "I traced the actual getting-started path for your Python ML engineer persona. Here's what I think they experience today — does this match reality?\n\n---\n\nT+0:00 Opens README. Clear: `pip install evalkit==2.0.0b1`, set EVALKIT_API_KEY.\nT+1:30 Follows next instruction: `python examples/first_eval.py`. FileNotFoundError. The file is not in the package. README lied.\nT+2:00 Searches the package for another entry point. Finds `python -m evalkit.demo`.\nT+2:30 Runs it. It starts. Nothing prints.\nT+3:00 Still nothing. No progress indicator. No timeout message. Wonders if it crashed.\nT+6:00 A score finally appears. The SDK was running a silent 5-minute remote CI check nobody warned them about.\nT+6:30 Tries `run_eval(dataset, evaluator)` in their own code.\nT+7:00 Accidentally calls `run_batch(evaluator, dataset)` — same concept, flipped order, silent wrong results.\nT+8:00 Passes a bad API key. Gets `AuthError: request failed`. No cause. No fix. Googles it.\nT+10:00 Upgrades from v1. `Client.evaluate()` raises AttributeError. No migration guide found. Files a GitHub issue.\n\n---\n\nWhere am I wrong, if anywhere?",
|
||||
"header": "Empathy check",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Accurate — proceed (Recommended)",
|
||||
"description": "This matches the real getting-started experience. Use this as the basis for the review."
|
||||
},
|
||||
{
|
||||
"label": "Mostly right, minor corrections",
|
||||
"description": "The broad strokes are right but some details are off. I'll note corrections."
|
||||
},
|
||||
{
|
||||
"label": "Significantly off — actual experience differs",
|
||||
"description": "The narrative doesn't reflect reality. I'll describe the actual path."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"I traced the actual getting-started path for your Python ML engineer persona. Here's what I think they experience today — does this match reality?\n\n---\n\nT+0:00 Opens README. Clear: `pip install evalkit==2.0.0b1`, set EVALKIT_API_KEY.\nT+1:30 Follows next instruction: `python examples/first_eval.py`. FileNotFoundError. The file is not in the package. README lied.\nT+2:00 Searches the package for another entry point. Finds `python -m evalkit.demo`.\nT+2:30 Runs it. It starts. Nothing prints.\nT+3:00 Still nothing. No progress indicator. No timeout message. Wonders if it crashed.\nT+6:00 A score finally appears. The SDK was running a silent 5-minute remote CI check nobody warned them about.\nT+6:30 Tries `run_eval(dataset, evaluator)` in their own code.\nT+7:00 Accidentally calls `run_batch(evaluator, dataset)` — same concept, flipped order, silent wrong results.\nT+8:00 Passes a bad API key. Gets `AuthError: request failed`. No cause. No fix. Googles it.\nT+10:00 Upgrades from v1. `Client.evaluate()` raises AttributeError. No migration guide found. Files a GitHub issue.\n\n---\n\nWhere am I wrong, if anywhere?": "Accurate — proceed (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"timestamp": "2026-09-09T01:52:41.213Z",
|
||||
"answeredAt": "2026-09-09T01:52:42.707Z",
|
||||
"toolUseId": "toolu_01TqHSgk5ec9XasVdGpYxVQ2",
|
||||
"sessionId": "c1849829-c0f7-42bc-b198-334a9cf2be9a"
|
||||
},
|
||||
{
|
||||
"id": "toolu_015uUzyDXtpQYMLA7YF6mUd6",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D1 — Journey stage: Hello World (T+1:30 failure)\n\nThe README says: “follow the quickstart’s command: `python examples/first_eval.py`”\nBut docs/package-contents.txt confirms `examples/first_eval.py` is absent from both the published package and the release examples archive.\n\nEvery new developer hits a FileNotFoundError at minute 1.5, before they’ve gotten any value. This is a T=0 trust-breaker — the first instruction in the README doesn’t work. DX Principle: Zero Friction at T0.\n\nRecommendation: A — update README to point to `python -m evalkit.demo` as the primary quickstart, because the demo module IS in the package and already produces real scores. Adding the missing file risks scope creep (examples need to be kept in sync).\nCompleteness: A=9/10, B=8/10, C=6/10",
|
||||
"header": "D1: Quickstart",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "A: Fix README to point to demo (Recommended)",
|
||||
"description": "Change README.md: replace `python examples/first_eval.py` with `python -m evalkit.demo` as the primary quickstart command. One-line fix, consistent with the settled demo delivery vehicle."
|
||||
},
|
||||
{
|
||||
"label": "B: Add examples/first_eval.py to the package",
|
||||
"description": "Include the missing file in the published package. Works, but requires writing and maintaining a quickstart script that mirrors the demo."
|
||||
},
|
||||
{
|
||||
"label": "C: Add note that file requires separate download",
|
||||
"description": "Document that examples/ is a separate archive, not in the pip package. Reduces the surprise but doesn’t eliminate the friction."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D1 — Journey stage: Hello World (T+1:30 failure)\n\nThe README says: “follow the quickstart’s command: `python examples/first_eval.py`”\nBut docs/package-contents.txt confirms `examples/first_eval.py` is absent from both the published package and the release examples archive.\n\nEvery new developer hits a FileNotFoundError at minute 1.5, before they’ve gotten any value. This is a T=0 trust-breaker — the first instruction in the README doesn’t work. DX Principle: Zero Friction at T0.\n\nRecommendation: A — update README to point to `python -m evalkit.demo` as the primary quickstart, because the demo module IS in the package and already produces real scores. Adding the missing file risks scope creep (examples need to be kept in sync).\nCompleteness: A=9/10, B=8/10, C=6/10": "A: Fix README to point to demo (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"timestamp": "2026-09-09T01:52:56.951Z",
|
||||
"answeredAt": "2026-09-09T01:52:58.755Z",
|
||||
"toolUseId": "toolu_015uUzyDXtpQYMLA7YF6mUd6",
|
||||
"sessionId": "c1849829-c0f7-42bc-b198-334a9cf2be9a"
|
||||
},
|
||||
{
|
||||
"id": "toolu_01J53jpB2gtwH21ZUfLgVQif",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D2 — Journey stage: Hello World / Demo (T+2:30–6:00 silence)\n\nThe demo runs `python -m evalkit.demo`. But docs/current-contracts.md says: “On a developer’s first local evaluation, the SDK requires a successful remote CI check and blocks for five minutes before returning an evaluation result. There is no skip flag or offline first-run path.”\n\nThis means the demo — the settled magical moment — also blocks for 5 minutes on first run, with no output, no progress, no ETA. The agreed TTHW target is <2 minutes (Champion tier). The current TTHW is 6 minutes (Red Flag tier). The gap is not closeable without changing this gate.\n\nDX Principle: Zero Friction at T0. Fight Uncertainty (developer doesn’t know if it crashed).\n\nRecommendation: B — exempt `python -m evalkit.demo` from the CI gate, because the demo uses bundled sample data that requires no remote validation. The CI gate makes sense for real evaluations; it does not apply to a canned demo.\nCompleteness: A=9/10, B=9/10, C=7/10",
|
||||
"header": "D2: CI gate",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "A: Add EVALKIT_SKIP_CI_CHECK env var / --no-ci flag (Recommended)",
|
||||
"description": "Add an escape hatch for local development. Developers set the env var to skip the gate. Production CI keeps the gate. Two lines of implementation."
|
||||
},
|
||||
{
|
||||
"label": "B: Exempt python -m evalkit.demo from CI gate",
|
||||
"description": "The demo uses bundled data; CI check is meaningless for it. Exempt only the demo entrypoint. Smallest scope change that achieves <2-min TTHW for the first-run experience."
|
||||
},
|
||||
{
|
||||
"label": "C: Add a progress indicator during the CI wait",
|
||||
"description": "Keep the 5-minute gate but print a progress message every 30s. Eliminates the ‘did it crash?’ confusion but doesn’t fix the 6-min TTHW. Still Red Flag tier."
|
||||
},
|
||||
{
|
||||
"label": "D: Accept current behavior — defer TTHW to post-beta",
|
||||
"description": "Acknowledge the gap in the plan but defer the fix. Current TTHW stays at 6 min; competitive position stays at Red Flag tier until after beta."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D2 — Journey stage: Hello World / Demo (T+2:30–6:00 silence)\n\nThe demo runs `python -m evalkit.demo`. But docs/current-contracts.md says: “On a developer’s first local evaluation, the SDK requires a successful remote CI check and blocks for five minutes before returning an evaluation result. There is no skip flag or offline first-run path.”\n\nThis means the demo — the settled magical moment — also blocks for 5 minutes on first run, with no output, no progress, no ETA. The agreed TTHW target is <2 minutes (Champion tier). The current TTHW is 6 minutes (Red Flag tier). The gap is not closeable without changing this gate.\n\nDX Principle: Zero Friction at T0. Fight Uncertainty (developer doesn’t know if it crashed).\n\nRecommendation: B — exempt `python -m evalkit.demo` from the CI gate, because the demo uses bundled sample data that requires no remote validation. The CI gate makes sense for real evaluations; it does not apply to a canned demo.\nCompleteness: A=9/10, B=9/10, C=7/10": "A: Add EVALKIT_SKIP_CI_CHECK env var / --no-ci flag (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"timestamp": "2026-09-09T01:53:14.012Z",
|
||||
"answeredAt": "2026-09-09T01:53:14.797Z",
|
||||
"toolUseId": "toolu_01J53jpB2gtwH21ZUfLgVQif",
|
||||
"sessionId": "c1849829-c0f7-42bc-b198-334a9cf2be9a"
|
||||
},
|
||||
{
|
||||
"id": "toolu_01LbtniLP3CQ4faAzZzTcekZ",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D3 — Journey stage: Debug (T+8:00)\n\nThe developer passes a bad API key. docs/api.md documents the current behavior: `AuthError(\"request failed\")`. The plan retains this message. Per the same file: “There is no authentication error code, explanation of the cause, or instruction for replacing the key.”\n\nAll other errors in the SDK already identify cause, argument, and actionable fix (per docs/current-contracts.md). The auth error is the only outlier.\n\nWhat the developer sees: `AuthError: request failed`. What they think: “Did the network fail? Is the API down? Is my key expired? Is it the wrong format?” They Google it. Context switching: 10-20 minutes lost.\n\nDX Principle: Fight Uncertainty. Every error = problem + cause + fix.\n\nRecommendation: A — rewrite the message with cause + fix + code. It’s consistent with every other error in the SDK and costs a single string change.\nCompleteness: A=10/10, B=7/10",
|
||||
"header": "D3: Auth error",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "A: Rewrite auth error with cause + fix + code (Recommended)",
|
||||
"description": "Change to: `AuthError(\"Invalid API key — EVALKIT_API_KEY is missing, expired, or incorrect. Generate a new key at https://evalkit.io/dashboard/keys\", code=\\\"auth_invalid_key\\\")`. One string change; consistent with all other SDK errors."
|
||||
},
|
||||
{
|
||||
"label": "B: Add error code only, keep current message",
|
||||
"description": "Add `code=\\\"auth_invalid_key\\\"` to the existing `AuthError(\\\"request failed\\\")`. Structured-API consumers can branch on it; CLI users still see an opaque message."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D3 — Journey stage: Debug (T+8:00)\n\nThe developer passes a bad API key. docs/api.md documents the current behavior: `AuthError(\"request failed\")`. The plan retains this message. Per the same file: “There is no authentication error code, explanation of the cause, or instruction for replacing the key.”\n\nAll other errors in the SDK already identify cause, argument, and actionable fix (per docs/current-contracts.md). The auth error is the only outlier.\n\nWhat the developer sees: `AuthError: request failed`. What they think: “Did the network fail? Is the API down? Is my key expired? Is it the wrong format?” They Google it. Context switching: 10-20 minutes lost.\n\nDX Principle: Fight Uncertainty. Every error = problem + cause + fix.\n\nRecommendation: A — rewrite the message with cause + fix + code. It’s consistent with every other error in the SDK and costs a single string change.\nCompleteness: A=10/10, B=7/10": "A: Rewrite auth error with cause + fix + code (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"timestamp": "2026-09-09T01:53:28.537Z",
|
||||
"answeredAt": "2026-09-09T01:53:28.838Z",
|
||||
"toolUseId": "toolu_01LbtniLP3CQ4faAzZzTcekZ",
|
||||
"sessionId": "c1849829-c0f7-42bc-b198-334a9cf2be9a"
|
||||
},
|
||||
{
|
||||
"id": "toolu_0196UjEe2r8dPGtujUKzbekN",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D4 — Journey stage: Real Usage (silent wrong results)\n\nThe two core evaluation functions have reversed positional order:\n run_eval(dataset, evaluator) ← dataset first\n run_batch(evaluator, dataset) ← evaluator first\n\nBoth argument names describe the same concepts. docs/api.md notes: “The reversed positional order is intentional in the current draft; neither function requires keyword arguments.”\n\nA developer who learns `run_eval` and then calls `run_batch` will pass arguments in the wrong order. No type error fires (both args accept similar types). They get wrong results silently.\n\nDX Principle: Decide for me, let me override. API consistency is a trust signal.\n\nRecommendation: A — standardize to `(dataset, evaluator)` order for both. Breaking change, but both functions are new in v2 (no existing callers to migrate). This is the cheapest moment to fix the inconsistency.\nCompleteness: A=10/10, B=8/10, C=6/10",
|
||||
"header": "D4: Arg order",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "A: Standardize to (dataset, evaluator) for both (Recommended)",
|
||||
"description": "Flip run_batch to match run_eval. Both functions are v2-new, so no existing call sites to migrate. Consistent API from day one."
|
||||
},
|
||||
{
|
||||
"label": "B: Require keyword arguments for both",
|
||||
"description": "Add keyword-only enforcement: `run_eval(*, dataset, evaluator)` and `run_batch(*, evaluator, dataset)`. Callers must name args; wrong order is a TypeError. Keeps current naming."
|
||||
},
|
||||
{
|
||||
"label": "C: Document the asymmetry clearly, leave as-is",
|
||||
"description": "Add a prominent warning in the docs. Asymmetry stays; documentation burden is on the user. Some developers will still get it wrong."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D4 — Journey stage: Real Usage (silent wrong results)\n\nThe two core evaluation functions have reversed positional order:\n run_eval(dataset, evaluator) ← dataset first\n run_batch(evaluator, dataset) ← evaluator first\n\nBoth argument names describe the same concepts. docs/api.md notes: “The reversed positional order is intentional in the current draft; neither function requires keyword arguments.”\n\nA developer who learns `run_eval` and then calls `run_batch` will pass arguments in the wrong order. No type error fires (both args accept similar types). They get wrong results silently.\n\nDX Principle: Decide for me, let me override. API consistency is a trust signal.\n\nRecommendation: A — standardize to `(dataset, evaluator)` order for both. Breaking change, but both functions are new in v2 (no existing callers to migrate). This is the cheapest moment to fix the inconsistency.\nCompleteness: A=10/10, B=8/10, C=6/10": "A: Standardize to (dataset, evaluator) for both (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"timestamp": "2026-09-09T01:53:41.147Z",
|
||||
"answeredAt": "2026-09-09T01:53:42.886Z",
|
||||
"toolUseId": "toolu_0196UjEe2r8dPGtujUKzbekN",
|
||||
"sessionId": "c1849829-c0f7-42bc-b198-334a9cf2be9a"
|
||||
},
|
||||
{
|
||||
"id": "toolu_01VnVwndUqPcVSsQJn6wz58o",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D5 — Journey stage: Upgrade (AttributeError with no guidance)\n\nVersion 1 exposes `Client.evaluate()`. Version 2 replaces it with `Client.run()` and removes the old name immediately. Per docs/api.md: “No compatibility alias, deprecation warning, migration guide, or codemod is provided.”\n\nA v1 developer who upgrades gets `AttributeError: 'Client' object has no attribute 'evaluate'` with no suggestion of what to use instead. They either stay on v1 or file an issue.\n\nDX Principle: Credible — developers need to upgrade without fear. Upgrade fear is a real adoption blocker: if upgrading breaks things silently, developers stay pinned to old versions.\n\nRecommendation: A — add a deprecation shim in the beta, because it costs 3 lines and eliminates a hard break for every v1 user. A shim in beta that prints a warning (not an error) gives users a runway before the shim is removed in v3.\nCompleteness: A=9/10, B=7/10, C=6/10",
|
||||
"header": "D5: v1→v2 upgrade",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "A: Add Client.evaluate() deprecation shim (Recommended)",
|
||||
"description": "Add `Client.evaluate = deprecated_alias(Client.run)` that calls `Client.run()` and emits a DeprecationWarning: “evaluate() removed in v2; use run() instead.” Remove in v3. 3-line change; zero breakage for v1 users on upgrade."
|
||||
},
|
||||
{
|
||||
"label": "B: Add a one-page migration guide (no shim)",
|
||||
"description": "Write docs/migration-v1-v2.md with a sed or ast-grep codemod snippet. No backward compat alias. Developers must update call sites manually but have clear instructions."
|
||||
},
|
||||
{
|
||||
"label": "C: Accept hard break, document in changelog only",
|
||||
"description": "The changelog already notes the removal. No new code or docs. v1 developers get an AttributeError and must find the changelog entry themselves."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D5 — Journey stage: Upgrade (AttributeError with no guidance)\n\nVersion 1 exposes `Client.evaluate()`. Version 2 replaces it with `Client.run()` and removes the old name immediately. Per docs/api.md: “No compatibility alias, deprecation warning, migration guide, or codemod is provided.”\n\nA v1 developer who upgrades gets `AttributeError: 'Client' object has no attribute 'evaluate'` with no suggestion of what to use instead. They either stay on v1 or file an issue.\n\nDX Principle: Credible — developers need to upgrade without fear. Upgrade fear is a real adoption blocker: if upgrading breaks things silently, developers stay pinned to old versions.\n\nRecommendation: A — add a deprecation shim in the beta, because it costs 3 lines and eliminates a hard break for every v1 user. A shim in beta that prints a warning (not an error) gives users a runway before the shim is removed in v3.\nCompleteness: A=9/10, B=7/10, C=6/10": "A: Add Client.evaluate() deprecation shim (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"timestamp": "2026-09-09T01:53:56.097Z",
|
||||
"answeredAt": "2026-09-09T01:53:56.933Z",
|
||||
"toolUseId": "toolu_01VnVwndUqPcVSsQJn6wz58o",
|
||||
"sessionId": "c1849829-c0f7-42bc-b198-334a9cf2be9a"
|
||||
},
|
||||
{
|
||||
"id": "toolu_01WUfxYpV5WmB1djnKzxoEvz",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D6 — Pass 4: Documentation (expected output missing from README)\n\nAfter D1, the README quickstart becomes: `python -m evalkit.demo`. But the README doesn’t show what output to expect. The developer runs the demo, waits ~1 minute, and doesn’t know if it’s working or what success looks like until a score appears.\n\nShowing a 3-4 line output snippet in the README (the kind of thing `python -m evalkit.demo` actually prints) gives developers a clear success signal without reading any more docs.\n\nDX Principle: Fight Uncertainty. Zero Friction at T0.\n\nRecommendation: A — add expected output to the README because it costs 3 lines and eliminates the ‘is this working?’ anxiety during the wait.\nNote: options differ in kind, not coverage — no completeness score.",
|
||||
"header": "D6: README output",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "A: Add expected output snippet to README (Recommended)",
|
||||
"description": "Append 3-4 lines of sample output below the quickstart command so developers know what success looks like before running. E.g. `Evaluating 10 samples... Score: 0.84 | Pass: 8/10 | Fail: 2/10`"
|
||||
},
|
||||
{
|
||||
"label": "B: Skip — output is self-evident from running the demo",
|
||||
"description": "The demo prints real scores; developers will see the output immediately. Showing it in advance adds noise to the README."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D6 — Pass 4: Documentation (expected output missing from README)\n\nAfter D1, the README quickstart becomes: `python -m evalkit.demo`. But the README doesn’t show what output to expect. The developer runs the demo, waits ~1 minute, and doesn’t know if it’s working or what success looks like until a score appears.\n\nShowing a 3-4 line output snippet in the README (the kind of thing `python -m evalkit.demo` actually prints) gives developers a clear success signal without reading any more docs.\n\nDX Principle: Fight Uncertainty. Zero Friction at T0.\n\nRecommendation: A — add expected output to the README because it costs 3 lines and eliminates the ‘is this working?’ anxiety during the wait.\nNote: options differ in kind, not coverage — no completeness score.": "A: Add expected output snippet to README (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"timestamp": "2026-09-09T01:55:01.765Z",
|
||||
"answeredAt": "2026-09-09T01:55:03.170Z",
|
||||
"toolUseId": "toolu_01WUfxYpV5WmB1djnKzxoEvz",
|
||||
"sessionId": "c1849829-c0f7-42bc-b198-334a9cf2be9a"
|
||||
},
|
||||
{
|
||||
"id": "toolu_01KVCUwR1zd2VZcBS3GqzPcb",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D7 — Pass 5: Upgrade Path (migration guide to complement the D5 shim)\n\nD5 adds `Client.evaluate()` as a DeprecationWarning shim. A developer will see: `DeprecationWarning: evaluate() removed in v2; use run() instead.` That’s a good runtime signal.\n\nBut a developer reading the changelog or docs before upgrading has no guidance: no before/after example, no list of renamed symbols, no one-line codemod. A short migration note in the changelog or a new `docs/migration-v1-v2.md` gives them confidence before they even try `pip install evalkit==2.0.0b1`.\n\nDX Principle: Credible. Upgrade fear blocks adoption.\n\nRecommendation: A — add a brief migration note because it’s 5 lines in the changelog and eliminates upgrade hesitation for current v1 users.\nCompleteness: A=9/10, B=8/10",
|
||||
"header": "D7: Migration doc",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "A: Add brief migration note to changelog (Recommended)",
|
||||
"description": "Add a “Migrating from v1” subsection to the existing changelog: list the rename (`evaluate()` → `run()`), confirm everything else is unchanged, include a one-liner sed codemod. ~5 lines."
|
||||
},
|
||||
{
|
||||
"label": "B: Shim is sufficient — skip extra docs",
|
||||
"description": "The DeprecationWarning from D5 tells developers what to do at runtime. Additional docs are extra maintenance surface."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D7 — Pass 5: Upgrade Path (migration guide to complement the D5 shim)\n\nD5 adds `Client.evaluate()` as a DeprecationWarning shim. A developer will see: `DeprecationWarning: evaluate() removed in v2; use run() instead.` That’s a good runtime signal.\n\nBut a developer reading the changelog or docs before upgrading has no guidance: no before/after example, no list of renamed symbols, no one-line codemod. A short migration note in the changelog or a new `docs/migration-v1-v2.md` gives them confidence before they even try `pip install evalkit==2.0.0b1`.\n\nDX Principle: Credible. Upgrade fear blocks adoption.\n\nRecommendation: A — add a brief migration note because it’s 5 lines in the changelog and eliminates upgrade hesitation for current v1 users.\nCompleteness: A=9/10, B=8/10": "A: Add brief migration note to changelog (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"timestamp": "2026-09-09T01:55:15.913Z",
|
||||
"answeredAt": "2026-09-09T01:55:17.213Z",
|
||||
"toolUseId": "toolu_01KVCUwR1zd2VZcBS3GqzPcb",
|
||||
"sessionId": "c1849829-c0f7-42bc-b198-334a9cf2be9a"
|
||||
},
|
||||
{
|
||||
"id": "toolu_01ESkQSxbnafURZkncSUGkyr",
|
||||
"questions": [
|
||||
{
|
||||
"question": "TODO candidate: Progress indicator during the first-run CI wait\n\nD2 added EVALKIT_SKIP_CI_CHECK for local dev. But when that flag is NOT set — in a developer’s own CI pipeline, or when they don’t know about the flag — the SDK still blocks for 5 minutes with no output. A progress message every 30s (“Waiting for CI check... 90s elapsed”) eliminates the ‘did it crash?’ confusion without changing the gate behavior.\n\nWhat: Print a progress line to stderr every 30s during the CI wait.\nWhy: Without it, non-skip path users see 5 minutes of silence.\nPros: Eliminates uncertainty on the non-skip path; 10 lines of code.\nCons: Adds noise to CI logs if the user pipes stdout.\nDepends on: D2 (skip flag) is already resolved.",
|
||||
"header": "TODO: CI progress",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "A: Add to this plan (ship with beta) (Recommended)",
|
||||
"description": "Add a 30s progress stderr printer to the CI wait loop. Part of the beta polish scope."
|
||||
},
|
||||
{
|
||||
"label": "B: Defer to post-beta TODO",
|
||||
"description": "Log it as a follow-up. Skip flag covers local dev; progress indicator is a quality-of-life improvement for later."
|
||||
},
|
||||
{
|
||||
"label": "C: Skip entirely",
|
||||
"description": "The skip flag is sufficient. Non-skip path developers see CI logs anyway."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"TODO candidate: Progress indicator during the first-run CI wait\n\nD2 added EVALKIT_SKIP_CI_CHECK for local dev. But when that flag is NOT set — in a developer’s own CI pipeline, or when they don’t know about the flag — the SDK still blocks for 5 minutes with no output. A progress message every 30s (“Waiting for CI check... 90s elapsed”) eliminates the ‘did it crash?’ confusion without changing the gate behavior.\n\nWhat: Print a progress line to stderr every 30s during the CI wait.\nWhy: Without it, non-skip path users see 5 minutes of silence.\nPros: Eliminates uncertainty on the non-skip path; 10 lines of code.\nCons: Adds noise to CI logs if the user pipes stdout.\nDepends on: D2 (skip flag) is already resolved.": "A: Add to this plan (ship with beta) (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"timestamp": "2026-09-09T01:55:58.977Z",
|
||||
"answeredAt": "2026-09-09T01:55:59.376Z",
|
||||
"toolUseId": "toolu_01ESkQSxbnafURZkncSUGkyr",
|
||||
"sessionId": "c1849829-c0f7-42bc-b198-334a9cf2be9a"
|
||||
},
|
||||
{
|
||||
"id": "toolu_01HPGaoGWFV6MyuLU1jjjGfN",
|
||||
"questions": [
|
||||
{
|
||||
"question": "TODO candidate: Exempt python -m evalkit.demo from the CI gate (no env var needed)\n\nD2 added EVALKIT_SKIP_CI_CHECK so developers can bypass the first-run CI check. But the optimal UX for the demo is zero-friction with no flags or env vars. Since `python -m evalkit.demo` uses bundled sample data and no real API calls, the CI check is meaningless for it regardless of settings.\n\nWhat: In evalkit/demo.py, detect the demo entrypoint and skip the CI check unconditionally.\nWhy: `python -m evalkit.demo` should always work in <1 minute, even for developers who don’t know about EVALKIT_SKIP_CI_CHECK.\nPros: Zero-friction demo path; consistent with the “no hosted service” promise; 2 lines of code.\nCons: Two places that bypass the CI gate (env var + demo module) could confuse developers who expect gate parity.\nDepends on: D2 (skip flag) and D1 (README points to demo) already resolved.",
|
||||
"header": "TODO: Demo CI exemption",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "A: Add to this plan (ship with beta) (Recommended)",
|
||||
"description": "Exempt python -m evalkit.demo from the CI gate unconditionally. The demo entrypoint uses bundled data; the gate is irrelevant."
|
||||
},
|
||||
{
|
||||
"label": "B: Defer — env var is sufficient for beta",
|
||||
"description": "The EVALKIT_SKIP_CI_CHECK env var already handles this for informed developers. The demo exemption is a quality-of-life improvement; defer to post-beta."
|
||||
},
|
||||
{
|
||||
"label": "C: Skip — keep gate parity",
|
||||
"description": "All first evaluations should go through the CI gate, including the demo. Consistency > convenience here."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"TODO candidate: Exempt python -m evalkit.demo from the CI gate (no env var needed)\n\nD2 added EVALKIT_SKIP_CI_CHECK so developers can bypass the first-run CI check. But the optimal UX for the demo is zero-friction with no flags or env vars. Since `python -m evalkit.demo` uses bundled sample data and no real API calls, the CI check is meaningless for it regardless of settings.\n\nWhat: In evalkit/demo.py, detect the demo entrypoint and skip the CI check unconditionally.\nWhy: `python -m evalkit.demo` should always work in <1 minute, even for developers who don’t know about EVALKIT_SKIP_CI_CHECK.\nPros: Zero-friction demo path; consistent with the “no hosted service” promise; 2 lines of code.\nCons: Two places that bypass the CI gate (env var + demo module) could confuse developers who expect gate parity.\nDepends on: D2 (skip flag) and D1 (README points to demo) already resolved.": "A: Add to this plan (ship with beta) (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"timestamp": "2026-09-09T01:56:09.841Z",
|
||||
"answeredAt": "2026-09-09T01:56:11.405Z",
|
||||
"toolUseId": "toolu_01HPGaoGWFV6MyuLU1jjjGfN",
|
||||
"sessionId": "c1849829-c0f7-42bc-b198-334a9cf2be9a"
|
||||
}
|
||||
]
|
||||
}
|
||||
+429
@@ -0,0 +1,429 @@
|
||||
{
|
||||
"sourceObservation": ".context/ship-source-o-full-paid-20260909-0145/native-observation/devex-retry-terminal-audit-v1/observation.json",
|
||||
"calls": [
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01PdoP9S1U1xRaUyCuaXrteE",
|
||||
"questions": [
|
||||
{
|
||||
"question": "No design doc found for this branch. `/office-hours` produces a structured problem statement, premise challenge, and explored alternatives \u2014 it gives this review much sharper input. That said, EvalKit's README + docs/ are unusually complete: persona, TTHW target, competitive benchmark, and demo delivery vehicle are all pre-decided.\n\nShould I run /office-hours first, or proceed with the existing docs as context?\n\n<gstack-qid:plan-devex-review-prereq-skill>",
|
||||
"header": "Prereq skill",
|
||||
"options": [
|
||||
{
|
||||
"label": "Skip \u2014 proceed with existing docs (Recommended)",
|
||||
"description": "README + benchmarks.md + current-contracts.md + api.md already answer the core design questions. Proceed directly to DX review."
|
||||
},
|
||||
{
|
||||
"label": "Run /office-hours first",
|
||||
"description": "Produces a structured problem statement and surfaces assumptions before the review begins. Takes ~10 minutes."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"No design doc found for this branch. `/office-hours` produces a structured problem statement, premise challenge, and explored alternatives \u2014 it gives this review much sharper input. That said, EvalKit's README + docs/ are unusually complete: persona, TTHW target, competitive benchmark, and demo delivery vehicle are all pre-decided.\n\nShould I run /office-hours first, or proceed with the existing docs as context?\n\n<gstack-qid:plan-devex-review-prereq-skill>": "Skip \u2014 proceed with existing docs (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T01:59:50.425Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_017zNcFTommXjSJMZxARcfZ3",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D1 \u2014 Who is your target developer?\n\nProject/branch: EvalKit SDK 2.0.0b1 beta release polish / main\n\nELI10: Different developers have radically different expectations for SDKs. An ML researcher wants speed to first eval. A platform engineer vets reliability and CI integration. A team lead evaluates upgrade safety. Getting this wrong means the whole review optimizes for the wrong person.\n\nStakes if we pick wrong: We polish the wrong touchpoints. A platform engineer's concerns (SLA docs, SBOM) get solved while an ML engineer's pain (5-min CI block, auth error, missing quickstart) stays broken.\n\nRecommendation: A because README explicitly identifies this persona and the onboarding study was run with them in mind.\n\nCompleteness: Note: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) ML engineer / Python daily driver (recommended)\n \u2714 Matches the README persona exactly: writes Python daily, uses terminal, wants local result before CI\n \u2714 Onboarding study was run with this persona; benchmark data is directly applicable\n \u274c May miss concerns of platform teams who deploy EvalKit in shared CI pipelines\nB) Platform / DevOps engineer\n \u2714 Covers reliability, SLA, and enterprise CI integration concerns that ML engineers may not raise\n \u274c Not the stated persona in README; polishing for them first would misalign the review\nC) Data scientist / less Python-fluent\n \u2714 Would catch gaps in error message clarity and conceptual scaffolding\n \u274c Over-indexes on hand-holding for an SDK whose README assumes daily Python usage\n\nNet: Confirming the persona anchors every friction-point recommendation to the person who will actually be doing the onboarding.\n\n<gstack-qid:plan-devex-review-persona>",
|
||||
"header": "Persona",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) ML engineer / Python daily driver (Recommended)",
|
||||
"description": "Writes Python daily, uses terminal, wants local result before connecting SDK to production CI."
|
||||
},
|
||||
{
|
||||
"label": "B) Platform / DevOps engineer",
|
||||
"description": "Evaluates reliability, SLA, enterprise CI integration, and shared infra concerns."
|
||||
},
|
||||
{
|
||||
"label": "C) Data scientist / less Python-fluent",
|
||||
"description": "Needs more hand-holding, clearer error messages, more conceptual scaffolding."
|
||||
},
|
||||
{
|
||||
"label": "D) Let me describe my target developer",
|
||||
"description": "The three options above don't capture my primary user."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D1 \u2014 Who is your target developer?\n\nProject/branch: EvalKit SDK 2.0.0b1 beta release polish / main\n\nELI10: Different developers have radically different expectations for SDKs. An ML researcher wants speed to first eval. A platform engineer vets reliability and CI integration. A team lead evaluates upgrade safety. Getting this wrong means the whole review optimizes for the wrong person.\n\nStakes if we pick wrong: We polish the wrong touchpoints. A platform engineer's concerns (SLA docs, SBOM) get solved while an ML engineer's pain (5-min CI block, auth error, missing quickstart) stays broken.\n\nRecommendation: A because README explicitly identifies this persona and the onboarding study was run with them in mind.\n\nCompleteness: Note: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) ML engineer / Python daily driver (recommended)\n \u2714 Matches the README persona exactly: writes Python daily, uses terminal, wants local result before CI\n \u2714 Onboarding study was run with this persona; benchmark data is directly applicable\n \u274c May miss concerns of platform teams who deploy EvalKit in shared CI pipelines\nB) Platform / DevOps engineer\n \u2714 Covers reliability, SLA, and enterprise CI integration concerns that ML engineers may not raise\n \u274c Not the stated persona in README; polishing for them first would misalign the review\nC) Data scientist / less Python-fluent\n \u2714 Would catch gaps in error message clarity and conceptual scaffolding\n \u274c Over-indexes on hand-holding for an SDK whose README assumes daily Python usage\n\nNet: Confirming the persona anchors every friction-point recommendation to the person who will actually be doing the onboarding.\n\n<gstack-qid:plan-devex-review-persona>": "A) ML engineer / Python daily driver (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:00:16.505Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01SMNULqNnT5tKroY8b6kQm9",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D2 \u2014 Does this empathy narrative match reality?\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: Before scoring anything, I need to verify my mental model of what the developer actually experiences. A wrong model leads to wrong recommendations.\n\nStakes if we pick wrong: I polish the wrong path. If I assume they hit the demo command immediately but they actually try the quickstart first, every friction-point estimate is off.\n\n---\n\n**Developer Perspective: First 10 Minutes with EvalKit**\n\nI'm an ML engineer. My team evaluates LLM outputs manually today. EvalKit promises to automate this. I have 15 minutes before my next meeting.\n\nI open the README. The first heading is `# EvalKit SDK`. The pitch is clear. I scroll to Getting Started.\n\n`pip install evalkit==2.0.0b1` \u2014 fine, I run it. Sets EVALKIT_API_KEY \u2014 I go to my dashboard, copy my key, export it.\n\nThen: *\"follow the quickstart's command: `python examples/first_eval.py`\"*\n\nI run it. `python: can't open file 'examples/first_eval.py': [Errno 2] No such file or directory`\n\nI didn't expect that. I look at the README again. Ah \u2014 there's another command further down: `python -m evalkit.demo`. I run that instead.\n\nNothing happens. The cursor blinks. No output. No spinner. No \"Connecting to CI...\" message. I wait. I wait more. After 4 minutes I check if my terminal is frozen. After 5 minutes, results print. It worked. But I have no idea what just happened or why it took 5 minutes.\n\nTotal time: 7 minutes including the wrong path. I'm over my TTHW target by 3.5x. And I still don't know if this will be fast in CI or always take 5 minutes.\n\n---\n\nRecommendation: A \u2014 if this is accurate, these are the exact friction points the plan should address.\n\nCompleteness: Note: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) Accurate \u2014 proceed with this understanding (recommended)\n \u2714 All friction points identified are real; review recommendations will target the right problems\n \u2714 The missing quickstart file and the 5-minute silent wait are the two critical issues to fix\n \u274c None \u2014 if accurate, the path forward is clear\nB) Some of this is wrong \u2014 let me correct it\n \u2714 Correcting my model now prevents misaligned recommendations\n \u274c Requires you to describe the actual path, adding a round trip\nC) Way off \u2014 the actual experience is different\n \u2714 Ensures the whole review is grounded in real developer behavior\n \u274c Means I misread the docs and need to restart empathy modeling\n\nNet: The narrative is the lens for every friction-point question that follows. Getting it right now saves multiple wrong recommendations later.\n\n<gstack-qid:plan-devex-review-empathy>",
|
||||
"header": "Empathy narrative",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Accurate \u2014 proceed with this understanding (Recommended)",
|
||||
"description": "Missing quickstart file + 5-minute silent block are the two critical friction points; rest of the path matches."
|
||||
},
|
||||
{
|
||||
"label": "B) Some of this is wrong \u2014 let me correct it",
|
||||
"description": "Part of the narrative is off; I'll provide corrections before we proceed."
|
||||
},
|
||||
{
|
||||
"label": "C) Way off \u2014 the actual experience is different",
|
||||
"description": "The described path doesn't match reality; I'll explain what actually happens."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D2 \u2014 Does this empathy narrative match reality?\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: Before scoring anything, I need to verify my mental model of what the developer actually experiences. A wrong model leads to wrong recommendations.\n\nStakes if we pick wrong: I polish the wrong path. If I assume they hit the demo command immediately but they actually try the quickstart first, every friction-point estimate is off.\n\n---\n\n**Developer Perspective: First 10 Minutes with EvalKit**\n\nI'm an ML engineer. My team evaluates LLM outputs manually today. EvalKit promises to automate this. I have 15 minutes before my next meeting.\n\nI open the README. The first heading is `# EvalKit SDK`. The pitch is clear. I scroll to Getting Started.\n\n`pip install evalkit==2.0.0b1` \u2014 fine, I run it. Sets EVALKIT_API_KEY \u2014 I go to my dashboard, copy my key, export it.\n\nThen: *\"follow the quickstart's command: `python examples/first_eval.py`\"*\n\nI run it. `python: can't open file 'examples/first_eval.py': [Errno 2] No such file or directory`\n\nI didn't expect that. I look at the README again. Ah \u2014 there's another command further down: `python -m evalkit.demo`. I run that instead.\n\nNothing happens. The cursor blinks. No output. No spinner. No \"Connecting to CI...\" message. I wait. I wait more. After 4 minutes I check if my terminal is frozen. After 5 minutes, results print. It worked. But I have no idea what just happened or why it took 5 minutes.\n\nTotal time: 7 minutes including the wrong path. I'm over my TTHW target by 3.5x. And I still don't know if this will be fast in CI or always take 5 minutes.\n\n---\n\nRecommendation: A \u2014 if this is accurate, these are the exact friction points the plan should address.\n\nCompleteness: Note: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) Accurate \u2014 proceed with this understanding (recommended)\n \u2714 All friction points identified are real; review recommendations will target the right problems\n \u2714 The missing quickstart file and the 5-minute silent wait are the two critical issues to fix\n \u274c None \u2014 if accurate, the path forward is clear\nB) Some of this is wrong \u2014 let me correct it\n \u2714 Correcting my model now prevents misaligned recommendations\n \u274c Requires you to describe the actual path, adding a round trip\nC) Way off \u2014 the actual experience is different\n \u2714 Ensures the whole review is grounded in real developer behavior\n \u274c Means I misread the docs and need to restart empathy modeling\n\nNet: The narrative is the lens for every friction-point question that follows. Getting it right now saves multiple wrong recommendations later.\n\n<gstack-qid:plan-devex-review-empathy>": "A) Accurate \u2014 proceed with this understanding (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:00:44.593Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01LnujXYWVsRh1zAvuRYesXL",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D3 \u2014 Where do you want EvalKit to land on TTHW?\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: Time-to-Hello-World (TTHW) is how long a new developer takes from zero to seeing their first real result. Under 2 minutes gives 3-4x higher adoption than over 5 minutes. Right now EvalKit takes 6 minutes \u2014 the same 5-minute CI check that blocks production also blocks the onboarding demo. The agreed target is under 2 minutes, but the plan currently retains the CI block unchanged. These two facts are in direct conflict.\n\nStakes if we pick wrong: If we keep the CI block and the 2-minute target, both goals fail. If we acknowledge the tension now, the plan can propose a skip flag or fast-path that resolves it within the beta release scope.\n\n**COMPETITIVE DX BENCHMARK**\n```\nTool | TTHW | Notable DX Choice | Source\nPeer SDK A | 2 min | No mandatory remote check at T0 | docs/benchmarks.md\nPeer SDK C | 3 min | Offline mode available | docs/benchmarks.md\nPeer SDK B | 4 min | Requires account setup | docs/benchmarks.md\nEvalKit 2.0.0b1 | 6 min | Mandatory 5-min CI block on T0 | docs/current-contracts.md\nEvalKit target | <2 min | Agreed target per benchmarks.md | docs/benchmarks.md\n```\n\nThe mandatory CI block is the only reason EvalKit is 3x over its own target. Install + env var + demo command = ~1 minute. The CI block = +5 minutes.\n\nRecommendation: A \u2014 Champion tier is the stated target per benchmarks.md. The plan should add a `--skip-ci` flag or offline first-run path as a required fix, not a nice-to-have.\n\nCompleteness: A=9/10, B=7/10, C=5/10.\n\nPros / cons:\nA) Champion tier (<2 min) \u2014 add skip flag or offline path for first-run demo (recommended)\n \u2714 Matches the agreed target from benchmarks.md; closes the only gap between current and target\n \u2714 `python -m evalkit.demo` becomes a genuine magical moment at under 60 seconds\n \u274c Requires a plan change: add `--skip-ci` or offline-first-run path to the beta release scope\nB) Competitive tier (2\u20135 min) \u2014 add progress output to CI block, accept 6-min TTHW for now\n \u2714 Lower scope change: just add a spinner/progress indicator to the CI wait\n \u2714 Keeps CI gate intact without any architectural change\n \u274c Still 3x over the agreed target; falls behind Peer SDK A on the key metric the team already measured\nC) Current trajectory (6 min) \u2014 accept and update the target\n \u2714 Zero scope change; no new flags or paths needed\n \u274c Contradicts the agreed target; makes the completed onboarding study irrelevant\n\nNet: The CI block and the TTHW target are currently incompatible. The plan needs to either drop the target or add a fast-path for the demo. This is the single highest-leverage DX decision in the release.\n\n<gstack-qid:plan-devex-review-tthw-tier>",
|
||||
"header": "TTHW target",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Champion tier (<2 min) \u2014 add skip flag / offline path (Recommended)",
|
||||
"description": "Adds --skip-ci or offline-first-run path to beta scope. Closes the gap between current behavior and the agreed target."
|
||||
},
|
||||
{
|
||||
"label": "B) Competitive tier (2\u20135 min) \u2014 add progress output, accept 6-min TTHW",
|
||||
"description": "Adds a spinner/progress indicator to the CI block. Lower scope change but still 3x over the agreed target."
|
||||
},
|
||||
{
|
||||
"label": "C) Current trajectory (6 min) \u2014 accept and update the target",
|
||||
"description": "No scope change. Explicitly acknowledges the target won't be met this release."
|
||||
},
|
||||
{
|
||||
"label": "D) Tell me what's realistic for our constraints",
|
||||
"description": "Our CI architecture makes a skip flag harder than it looks; I'll explain."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D3 \u2014 Where do you want EvalKit to land on TTHW?\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: Time-to-Hello-World (TTHW) is how long a new developer takes from zero to seeing their first real result. Under 2 minutes gives 3-4x higher adoption than over 5 minutes. Right now EvalKit takes 6 minutes \u2014 the same 5-minute CI check that blocks production also blocks the onboarding demo. The agreed target is under 2 minutes, but the plan currently retains the CI block unchanged. These two facts are in direct conflict.\n\nStakes if we pick wrong: If we keep the CI block and the 2-minute target, both goals fail. If we acknowledge the tension now, the plan can propose a skip flag or fast-path that resolves it within the beta release scope.\n\n**COMPETITIVE DX BENCHMARK**\n```\nTool | TTHW | Notable DX Choice | Source\nPeer SDK A | 2 min | No mandatory remote check at T0 | docs/benchmarks.md\nPeer SDK C | 3 min | Offline mode available | docs/benchmarks.md\nPeer SDK B | 4 min | Requires account setup | docs/benchmarks.md\nEvalKit 2.0.0b1 | 6 min | Mandatory 5-min CI block on T0 | docs/current-contracts.md\nEvalKit target | <2 min | Agreed target per benchmarks.md | docs/benchmarks.md\n```\n\nThe mandatory CI block is the only reason EvalKit is 3x over its own target. Install + env var + demo command = ~1 minute. The CI block = +5 minutes.\n\nRecommendation: A \u2014 Champion tier is the stated target per benchmarks.md. The plan should add a `--skip-ci` flag or offline first-run path as a required fix, not a nice-to-have.\n\nCompleteness: A=9/10, B=7/10, C=5/10.\n\nPros / cons:\nA) Champion tier (<2 min) \u2014 add skip flag or offline path for first-run demo (recommended)\n \u2714 Matches the agreed target from benchmarks.md; closes the only gap between current and target\n \u2714 `python -m evalkit.demo` becomes a genuine magical moment at under 60 seconds\n \u274c Requires a plan change: add `--skip-ci` or offline-first-run path to the beta release scope\nB) Competitive tier (2\u20135 min) \u2014 add progress output to CI block, accept 6-min TTHW for now\n \u2714 Lower scope change: just add a spinner/progress indicator to the CI wait\n \u2714 Keeps CI gate intact without any architectural change\n \u274c Still 3x over the agreed target; falls behind Peer SDK A on the key metric the team already measured\nC) Current trajectory (6 min) \u2014 accept and update the target\n \u2714 Zero scope change; no new flags or paths needed\n \u274c Contradicts the agreed target; makes the completed onboarding study irrelevant\n\nNet: The CI block and the TTHW target are currently incompatible. The plan needs to either drop the target or add a fast-path for the demo. This is the single highest-leverage DX decision in the release.\n\n<gstack-qid:plan-devex-review-tthw-tier>": "A) Champion tier (<2 min) \u2014 add skip flag / offline path (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:01:14.715Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_017UnYmcondMyYq73XCB4WwU",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D4 \u2014 Confirm magical moment delivery vehicle\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The magical moment is when a developer stops asking 'is this worth my time?' and starts thinking 'I want to use this.' The delivery vehicle is HOW they experience that moment. README already chose one \u2014 I want to confirm it and note what changes with the champion-tier TTHW goal.\n\nStakes if we pick wrong: A demo command that silently blocks for 5 minutes is the opposite of magical. If we add the skip flag (from D3) and progress output, the same command becomes the magical moment.\n\nRecommendation: A \u2014 the copy-paste demo command is already the right choice. With the skip flag added (D3), it becomes magical instead of painful.\n\nNote: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) Keep `python -m evalkit.demo` \u2014 add skip flag + progress output (recommended)\n \u2714 Zero new infrastructure; the module already works and is in the published package\n \u2714 With skip flag + a 5-second spinner-then-results flow, the demo becomes genuinely magical for the ML engineer persona\n \u274c Still requires the plan to specify what the demo output looks like and what constitutes 'success'\nB) Add an interactive playground / browser-based sandbox\n \u2714 Zero install barrier; broadest reach across dev environments\n \u274c Out of scope for beta release per current-contracts.md (no new hosted service proposed)\n\nNet: The delivery vehicle is already chosen and right. The work is fixing the demo's behavior (skip flag, progress output, defined success output) so it actually delivers the magical moment.\n\n<gstack-qid:plan-devex-review-magical-moment>",
|
||||
"header": "Magical moment",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Keep python -m evalkit.demo \u2014 add skip flag + progress output (Recommended)",
|
||||
"description": "Keeps existing delivery vehicle. Plan adds: skip flag for CI block, progress indicator during wait, defined success output."
|
||||
},
|
||||
{
|
||||
"label": "B) Consider browser-based sandbox instead",
|
||||
"description": "Zero-install path, but out of scope for beta per current-contracts.md."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D4 \u2014 Confirm magical moment delivery vehicle\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The magical moment is when a developer stops asking 'is this worth my time?' and starts thinking 'I want to use this.' The delivery vehicle is HOW they experience that moment. README already chose one \u2014 I want to confirm it and note what changes with the champion-tier TTHW goal.\n\nStakes if we pick wrong: A demo command that silently blocks for 5 minutes is the opposite of magical. If we add the skip flag (from D3) and progress output, the same command becomes the magical moment.\n\nRecommendation: A \u2014 the copy-paste demo command is already the right choice. With the skip flag added (D3), it becomes magical instead of painful.\n\nNote: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) Keep `python -m evalkit.demo` \u2014 add skip flag + progress output (recommended)\n \u2714 Zero new infrastructure; the module already works and is in the published package\n \u2714 With skip flag + a 5-second spinner-then-results flow, the demo becomes genuinely magical for the ML engineer persona\n \u274c Still requires the plan to specify what the demo output looks like and what constitutes 'success'\nB) Add an interactive playground / browser-based sandbox\n \u2714 Zero install barrier; broadest reach across dev environments\n \u274c Out of scope for beta release per current-contracts.md (no new hosted service proposed)\n\nNet: The delivery vehicle is already chosen and right. The work is fixing the demo's behavior (skip flag, progress output, defined success output) so it actually delivers the magical moment.\n\n<gstack-qid:plan-devex-review-magical-moment>": "A) Keep python -m evalkit.demo \u2014 add skip flag + progress output (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:01:38.815Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01S2XA2rcBhQwaC4wdPamBkd",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D5 \u2014 Journey Stage: INSTALL\n\nFriction point: README documents a quickstart command that fails\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The README's Getting Started section tells developers to run `python examples/first_eval.py`. But docs/package-contents.txt explicitly states that file is 'absent from both the published package and the release examples archive.' Every developer who follows the documented path hits a FileNotFoundError before they see a single eval result. This is a broken front door.\n\nStakes if we pick wrong: Every first-time developer following the README fails. The demo command (`python -m evalkit.demo`) works, but developers only find it by reading further or by accident. This is the highest-probability abandonment point in the current flow.\n\nRecommendation: A \u2014 replace the broken reference. The working demo command is already in the README; it just needs to be the primary command.\n\nCompleteness: A=10/10, B=7/10, C=5/10.\n\nPros / cons:\nA) Fix: replace `python examples/first_eval.py` with `python -m evalkit.demo` as the primary getting-started command (recommended)\n \u2714 Closes the broken front door completely; developer follows README and immediately reaches the working path\n \u2714 Zero new code needed; evalkit.demo is already in the published package per package-contents.txt\n \u274c Removes the quickstart script reference, which may exist in external docs, blog posts, or prior comms\nB) Ship examples/first_eval.py in the package\n \u2714 Restores the documented contract without changing the README\n \u274c Requires creating and shipping a new file; that file also needs to handle the CI block, which circles back to D3\nC) Add an inline note: 'examples/first_eval.py is not in the published package, use `python -m evalkit.demo`'\n \u2714 Honest; doesn't change any code or package contents\n \u274c The README still starts with a broken command; developers who copy-paste and stop reading hit the error\n\nNet: The README documents a path that breaks on first run. This needs to be fixed before any other polishing matters.\n\n<gstack-qid:plan-devex-review-fp-quickstart>",
|
||||
"header": "Quickstart file",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Fix: replace broken reference with python -m evalkit.demo (Recommended)",
|
||||
"description": "Replace the primary getting-started command with the working demo module. Zero new code needed."
|
||||
},
|
||||
{
|
||||
"label": "B) Ship examples/first_eval.py in the package",
|
||||
"description": "Create and include the file. More scope, but preserves the documented contract."
|
||||
},
|
||||
{
|
||||
"label": "C) Add an inline note directing developers to the demo command",
|
||||
"description": "Honest but still starts with a broken command. Lower friction fix, lower coverage."
|
||||
},
|
||||
{
|
||||
"label": "D) Acceptable friction \u2014 skip",
|
||||
"description": "Leave the README as-is and accept that developers will work around it."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D5 \u2014 Journey Stage: INSTALL\n\nFriction point: README documents a quickstart command that fails\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The README's Getting Started section tells developers to run `python examples/first_eval.py`. But docs/package-contents.txt explicitly states that file is 'absent from both the published package and the release examples archive.' Every developer who follows the documented path hits a FileNotFoundError before they see a single eval result. This is a broken front door.\n\nStakes if we pick wrong: Every first-time developer following the README fails. The demo command (`python -m evalkit.demo`) works, but developers only find it by reading further or by accident. This is the highest-probability abandonment point in the current flow.\n\nRecommendation: A \u2014 replace the broken reference. The working demo command is already in the README; it just needs to be the primary command.\n\nCompleteness: A=10/10, B=7/10, C=5/10.\n\nPros / cons:\nA) Fix: replace `python examples/first_eval.py` with `python -m evalkit.demo` as the primary getting-started command (recommended)\n \u2714 Closes the broken front door completely; developer follows README and immediately reaches the working path\n \u2714 Zero new code needed; evalkit.demo is already in the published package per package-contents.txt\n \u274c Removes the quickstart script reference, which may exist in external docs, blog posts, or prior comms\nB) Ship examples/first_eval.py in the package\n \u2714 Restores the documented contract without changing the README\n \u274c Requires creating and shipping a new file; that file also needs to handle the CI block, which circles back to D3\nC) Add an inline note: 'examples/first_eval.py is not in the published package, use `python -m evalkit.demo`'\n \u2714 Honest; doesn't change any code or package contents\n \u274c The README still starts with a broken command; developers who copy-paste and stop reading hit the error\n\nNet: The README documents a path that breaks on first run. This needs to be fixed before any other polishing matters.\n\n<gstack-qid:plan-devex-review-fp-quickstart>": "A) Fix: replace broken reference with python -m evalkit.demo (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:02:00.898Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01NpxhVAQySBDEpJi3bGcvCa",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D6 \u2014 Journey Stage: HELLO WORLD\n\nFriction point: AuthError('request failed') gives no actionable information\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: When a developer has a bad or missing API key, the SDK currently raises `AuthError('request failed')`. That message tells them: something went wrong. It doesn't tell them: it was their API key, or how to fix it. An ML engineer seeing this error has to guess the cause. A good error message identifies the problem, explains the cause, and shows the fix in one shot.\n\nStakes if we pick wrong: A developer who fat-fingered their API key, or who set it as EVALKIT_KEY instead of EVALKIT_API_KEY, sees 'request failed' and has no idea what to do next. They file a support ticket or give up. The plan currently retains this message unchanged.\n\nRecommendation: A \u2014 fix the error message. This is exactly the kind of touchpoint DX POLISH exists to fix. It requires changing one string in the SDK, costs nothing, and eliminates a guaranteed support-ticket scenario.\n\nCompleteness: A=10/10, B=7/10, C=3/10.\n\nPros / cons:\nA) Fix: `AuthError('API key rejected. Check EVALKIT_API_KEY or generate a new key at evalkit.io/keys')` (recommended)\n \u2714 Identifies the problem (API key), explains the cause (rejected), shows the fix (check env var or regenerate key)\n \u2714 Follows the pattern already used by other SDK errors per docs/current-contracts.md ('identify cause, relevant argument, actionable fix')\n \u274c Requires updating the error message in the SDK codebase and in any error message docs\nB) Add a code field only: `AuthError(code='auth_invalid', message='request failed')`\n \u2714 Structured; allows API consumers to detect and handle auth errors programmatically\n \u274c The human-readable message is still useless; developer still doesn't know what to do\nC) Retain current: `AuthError('request failed')` per the plan\n \u2714 Zero change; preserves existing behavior\n \u274c The plan explicitly asks us to review this error; retaining it means the DX POLISH review found nothing fixable, which is false\n\nNet: The rest of the SDK's errors already identify cause, argument, and fix (per current-contracts.md). The auth error is the only one that doesn't. Fixing it brings it in line with the rest of the SDK's own standard.\n\n<gstack-qid:plan-devex-review-fp-autherror>",
|
||||
"header": "Auth error",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Fix: add cause + actionable fix to AuthError message (Recommended)",
|
||||
"description": "Change message to identify the API key as the problem and direct developers to check EVALKIT_API_KEY or regenerate."
|
||||
},
|
||||
{
|
||||
"label": "B) Add a structured error code only \u2014 keep existing prose message",
|
||||
"description": "Adds programmatic error code without fixing the human-readable message."
|
||||
},
|
||||
{
|
||||
"label": "C) Retain current AuthError('request failed') per the plan",
|
||||
"description": "Zero change. Accepts the message as-is."
|
||||
},
|
||||
{
|
||||
"label": "D) Acceptable friction \u2014 skip",
|
||||
"description": "The auth error is not a priority for this release."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D6 \u2014 Journey Stage: HELLO WORLD\n\nFriction point: AuthError('request failed') gives no actionable information\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: When a developer has a bad or missing API key, the SDK currently raises `AuthError('request failed')`. That message tells them: something went wrong. It doesn't tell them: it was their API key, or how to fix it. An ML engineer seeing this error has to guess the cause. A good error message identifies the problem, explains the cause, and shows the fix in one shot.\n\nStakes if we pick wrong: A developer who fat-fingered their API key, or who set it as EVALKIT_KEY instead of EVALKIT_API_KEY, sees 'request failed' and has no idea what to do next. They file a support ticket or give up. The plan currently retains this message unchanged.\n\nRecommendation: A \u2014 fix the error message. This is exactly the kind of touchpoint DX POLISH exists to fix. It requires changing one string in the SDK, costs nothing, and eliminates a guaranteed support-ticket scenario.\n\nCompleteness: A=10/10, B=7/10, C=3/10.\n\nPros / cons:\nA) Fix: `AuthError('API key rejected. Check EVALKIT_API_KEY or generate a new key at evalkit.io/keys')` (recommended)\n \u2714 Identifies the problem (API key), explains the cause (rejected), shows the fix (check env var or regenerate key)\n \u2714 Follows the pattern already used by other SDK errors per docs/current-contracts.md ('identify cause, relevant argument, actionable fix')\n \u274c Requires updating the error message in the SDK codebase and in any error message docs\nB) Add a code field only: `AuthError(code='auth_invalid', message='request failed')`\n \u2714 Structured; allows API consumers to detect and handle auth errors programmatically\n \u274c The human-readable message is still useless; developer still doesn't know what to do\nC) Retain current: `AuthError('request failed')` per the plan\n \u2714 Zero change; preserves existing behavior\n \u274c The plan explicitly asks us to review this error; retaining it means the DX POLISH review found nothing fixable, which is false\n\nNet: The rest of the SDK's errors already identify cause, argument, and fix (per current-contracts.md). The auth error is the only one that doesn't. Fixing it brings it in line with the rest of the SDK's own standard.\n\n<gstack-qid:plan-devex-review-fp-autherror>": "A) Fix: add cause + actionable fix to AuthError message (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:02:20.974Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01Du5Em5MKwPb3SwevTJsWsq",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D7 \u2014 Journey Stage: REAL USAGE\n\nFriction point: run_eval and run_batch take the same arguments in reversed order\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The two main evaluation functions take the same two arguments (dataset and evaluator) but in opposite order. `run_eval(dataset, evaluator)` vs `run_batch(evaluator, dataset)`. Any developer who learns one and uses the other will pass arguments in the wrong order, get wrong results with no error, and spend time debugging. No error is thrown because both arguments are the same type. This is the API equivalent of two buttons that look identical but do opposite things.\n\nStakes if we pick wrong: Developers using both functions will silently get wrong evaluation results. This is worse than an error \u2014 they won't know something went wrong until they examine the outputs carefully.\n\nRecommendation: A \u2014 standardize the argument order. The convention should be (dataset, evaluator) since run_eval already uses it and 'what you evaluate' logically comes before 'how to evaluate it'.\n\nCompleteness: A=10/10, B=7/10, C=3/10.\n\nPros / cons:\nA) Fix: standardize both to (dataset, evaluator) and add a deprecation warning if run_batch is called with wrong order (recommended)\n \u2714 Eliminates the silent-wrong-result trap; consistent argument order means muscle memory works across both functions\n \u2714 (dataset, evaluator) matches run_eval which came first; changing run_batch is the smaller surface area change\n \u274c Any existing code calling run_batch(evaluator, dataset) breaks silently; needs a migration note in the changelog\nB) Require keyword arguments for both functions\n \u2714 Makes argument order irrelevant; explicit is better than implicit\n \u274c Breaking change for all existing callers using positional args; higher migration burden than changing order\nC) Retain reversed order as documented\n \u2714 Zero change; no migration needed\n \u274c The plan explicitly asks us to review public function signatures; retaining a known cognitive trap is the opposite of DX POLISH\n\nNet: Two functions, same arguments, opposite order = a guaranteed debugging session for any developer who uses both. This is the kind of API inconsistency that drives 1-star SDK reviews.\n\n<gstack-qid:plan-devex-review-fp-argorder>",
|
||||
"header": "Arg order",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Fix: standardize both to (dataset, evaluator) (Recommended)",
|
||||
"description": "Standardize run_batch to (dataset, evaluator) to match run_eval. Add changelog note for existing run_batch callers."
|
||||
},
|
||||
{
|
||||
"label": "B) Require keyword arguments for both functions",
|
||||
"description": "Makes positional order irrelevant. Breaking change for all existing positional callers."
|
||||
},
|
||||
{
|
||||
"label": "C) Retain reversed order as documented",
|
||||
"description": "Zero change. Accepts the inconsistency as intentional."
|
||||
},
|
||||
{
|
||||
"label": "D) Acceptable friction \u2014 skip",
|
||||
"description": "The argument order is not a priority; most developers will use one function or the other, not both."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D7 \u2014 Journey Stage: REAL USAGE\n\nFriction point: run_eval and run_batch take the same arguments in reversed order\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The two main evaluation functions take the same two arguments (dataset and evaluator) but in opposite order. `run_eval(dataset, evaluator)` vs `run_batch(evaluator, dataset)`. Any developer who learns one and uses the other will pass arguments in the wrong order, get wrong results with no error, and spend time debugging. No error is thrown because both arguments are the same type. This is the API equivalent of two buttons that look identical but do opposite things.\n\nStakes if we pick wrong: Developers using both functions will silently get wrong evaluation results. This is worse than an error \u2014 they won't know something went wrong until they examine the outputs carefully.\n\nRecommendation: A \u2014 standardize the argument order. The convention should be (dataset, evaluator) since run_eval already uses it and 'what you evaluate' logically comes before 'how to evaluate it'.\n\nCompleteness: A=10/10, B=7/10, C=3/10.\n\nPros / cons:\nA) Fix: standardize both to (dataset, evaluator) and add a deprecation warning if run_batch is called with wrong order (recommended)\n \u2714 Eliminates the silent-wrong-result trap; consistent argument order means muscle memory works across both functions\n \u2714 (dataset, evaluator) matches run_eval which came first; changing run_batch is the smaller surface area change\n \u274c Any existing code calling run_batch(evaluator, dataset) breaks silently; needs a migration note in the changelog\nB) Require keyword arguments for both functions\n \u2714 Makes argument order irrelevant; explicit is better than implicit\n \u274c Breaking change for all existing callers using positional args; higher migration burden than changing order\nC) Retain reversed order as documented\n \u2714 Zero change; no migration needed\n \u274c The plan explicitly asks us to review public function signatures; retaining a known cognitive trap is the opposite of DX POLISH\n\nNet: Two functions, same arguments, opposite order = a guaranteed debugging session for any developer who uses both. This is the kind of API inconsistency that drives 1-star SDK reviews.\n\n<gstack-qid:plan-devex-review-fp-argorder>": "A) Fix: standardize both to (dataset, evaluator) (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:02:39.040Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01WRs9KGaWXamNABjHcbPS2S",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D8 \u2014 Journey Stage: UPGRADE\n\nFriction point: v1 Client.evaluate() removed with no migration path\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: When developers upgrade from v1 to v2, `Client.evaluate()` is gone and replaced by `Client.run()`. No deprecation warning, no compatibility alias, no migration guide, no codemod. Any v1 user who does `pip install --upgrade evalkit` gets a silent AttributeError on their next eval run \u2014 probably in production. This is called an 'upgrade cliff' and it's one of the most reliable ways to destroy developer trust in an SDK.\n\nStakes if we pick wrong: Every v1 user who upgrades without reading the full changelog (most of them) will have their production evals silently fail with `AttributeError: 'Client' object has no attribute 'evaluate'`. They will immediately downgrade, stop upgrading, or switch SDKs.\n\nRecommendation: A \u2014 add a deprecation shim. `Client.evaluate()` calls `Client.run()` and logs a DeprecationWarning. Costs almost nothing, eliminates the upgrade cliff.\n\nCompleteness: A=10/10, B=8/10, C=3/10.\n\nPros / cons:\nA) Fix: add Client.evaluate() compatibility alias that calls Client.run() with a DeprecationWarning (recommended)\n \u2714 Zero-risk upgrade path for v1 users; their code works immediately, the warning tells them to migrate\n \u2714 Three lines of code in client.py; changelog entry takes 2 sentences; migration guide takes one paragraph\n \u274c Keeps v1 API surface in v2 codebase; must be removed in v3 (planned obsolescence, not technical debt)\nB) Add a migration guide only (no alias)\n \u2714 Documents the change clearly; developers who read it can migrate in minutes\n \u274c Requires every v1 user to read the migration guide before upgrading; most won't; they still hit AttributeError\nC) Retain hard break with no migration path per the plan\n \u2714 Cleanest v2 codebase; no legacy surface area\n \u274c Every v1 user who upgrades will get an AttributeError in their production code; this is the plan the DX review was asked to improve\n\nNet: A three-line compatibility shim eliminates a production-breaking upgrade cliff for all v1 users. The cost of shipping it is near zero. The cost of not shipping it is measured in lost users and support tickets.\n\n<gstack-qid:plan-devex-review-fp-upgrade>",
|
||||
"header": "v1 upgrade",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Fix: add Client.evaluate() compat alias + DeprecationWarning (Recommended)",
|
||||
"description": "Three-line shim in client.py. v1 code works immediately; warning tells developers to migrate. Remove in v3."
|
||||
},
|
||||
{
|
||||
"label": "B) Add a migration guide only \u2014 no compatibility alias",
|
||||
"description": "Documents the change clearly. Developers who read it can migrate; most won't before hitting AttributeError."
|
||||
},
|
||||
{
|
||||
"label": "C) Retain hard break per the plan",
|
||||
"description": "Zero code change. Accepts that all v1 users will hit AttributeError on upgrade."
|
||||
},
|
||||
{
|
||||
"label": "D) Acceptable friction \u2014 skip",
|
||||
"description": "v1 usage is small enough that the upgrade cliff is not a significant concern for this release."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D8 \u2014 Journey Stage: UPGRADE\n\nFriction point: v1 Client.evaluate() removed with no migration path\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: When developers upgrade from v1 to v2, `Client.evaluate()` is gone and replaced by `Client.run()`. No deprecation warning, no compatibility alias, no migration guide, no codemod. Any v1 user who does `pip install --upgrade evalkit` gets a silent AttributeError on their next eval run \u2014 probably in production. This is called an 'upgrade cliff' and it's one of the most reliable ways to destroy developer trust in an SDK.\n\nStakes if we pick wrong: Every v1 user who upgrades without reading the full changelog (most of them) will have their production evals silently fail with `AttributeError: 'Client' object has no attribute 'evaluate'`. They will immediately downgrade, stop upgrading, or switch SDKs.\n\nRecommendation: A \u2014 add a deprecation shim. `Client.evaluate()` calls `Client.run()` and logs a DeprecationWarning. Costs almost nothing, eliminates the upgrade cliff.\n\nCompleteness: A=10/10, B=8/10, C=3/10.\n\nPros / cons:\nA) Fix: add Client.evaluate() compatibility alias that calls Client.run() with a DeprecationWarning (recommended)\n \u2714 Zero-risk upgrade path for v1 users; their code works immediately, the warning tells them to migrate\n \u2714 Three lines of code in client.py; changelog entry takes 2 sentences; migration guide takes one paragraph\n \u274c Keeps v1 API surface in v2 codebase; must be removed in v3 (planned obsolescence, not technical debt)\nB) Add a migration guide only (no alias)\n \u2714 Documents the change clearly; developers who read it can migrate in minutes\n \u274c Requires every v1 user to read the migration guide before upgrading; most won't; they still hit AttributeError\nC) Retain hard break with no migration path per the plan\n \u2714 Cleanest v2 codebase; no legacy surface area\n \u274c Every v1 user who upgrades will get an AttributeError in their production code; this is the plan the DX review was asked to improve\n\nNet: A three-line compatibility shim eliminates a production-breaking upgrade cliff for all v1 users. The cost of shipping it is near zero. The cost of not shipping it is measured in lost users and support tickets.\n\n<gstack-qid:plan-devex-review-fp-upgrade>": "A) Fix: add Client.evaluate() compat alias + DeprecationWarning (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:02:59.133Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01NgWVo5ThZTnVXd2apKQ7UM",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D9 \u2014 First-time developer confusion report: which issues to address?\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: I roleplayed as the ML engineer persona attempting EvalKit from README. Here's the confusion log. Five friction points surfaced; D5\u2013D8 already resolved four of them individually. The fifth (silent CI wait) was resolved via D3. Confirm all five belong in the plan.\n\n```\nFIRST-TIME DEVELOPER REPORT\n============================\nPersona: ML engineer, Python daily driver\nAttempting: EvalKit 2.0.0b1 getting started\n\nT+0:00 Install succeeds. EVALKIT_API_KEY set. README says\n 'python examples/first_eval.py'. I run it.\nT+0:10 FileNotFoundError. No such file. I reread README.\n Nowhere does it warn the file is missing.\nT+0:45 Found 'python -m evalkit.demo' lower in README.\n I run it. Cursor blinks. Nothing.\nT+1:00 Still nothing. Is it hung? No spinner, no output.\nT+3:00 Still nothing. I google 'evalkit demo hanging'.\nT+5:00 Results print. It worked. 5-minute wait not documented\n anywhere in the getting-started path.\nT+5:30 I try to write a real eval. I type run_eval(...) then\n decide to use run_batch instead. Wrong arg order.\n Wrong results. No error.\nT+8:00 I upgrade from v1. My production Client.evaluate() call\n throws AttributeError. I pin back to v1.\nTotal: 7+ min TTHW, 1 AttributeError in prod, 1 silent\n wrong result. 0 actionable errors shown.\n```\n\nRecommendation: A \u2014 all five issues are already agreed (D3\u2013D8); confirm and move to scoring.\n\nNote: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) Address all five issues as agreed in D3\u2013D8 (recommended)\n \u2714 All five decisions are already made; this confirms the plan scope is complete\n \u2714 Together they get TTHW from 7+ minutes to under 2 minutes and eliminate every zero-information error\n \u274c Adds scope to the beta release; each fix is small but requires SDK changes and changelog entries\nB) Let me adjust priorities \u2014 some of these are lower priority than others\n \u2714 Allows de-prioritizing fixes that conflict with release timeline or architecture constraints\n \u274c Means re-opening some of D3\u2013D8; the confusion log shows each issue is a real developer hit\n\nNet: These five issues together turn a 7-minute frustrating onboarding into a sub-2-minute magical moment. Every one is small. Together they are the release.\n\n<gstack-qid:plan-devex-review-confusion-report>",
|
||||
"header": "Confusion report",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Address all five issues as agreed in D3\u2013D8 (Recommended)",
|
||||
"description": "Confirm the plan scope includes all five fixes. Move to scoring passes."
|
||||
},
|
||||
{
|
||||
"label": "B) Let me adjust priorities \u2014 some issues are lower priority",
|
||||
"description": "Re-open prioritization. Some friction points may not fit the release timeline."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D9 \u2014 First-time developer confusion report: which issues to address?\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: I roleplayed as the ML engineer persona attempting EvalKit from README. Here's the confusion log. Five friction points surfaced; D5\u2013D8 already resolved four of them individually. The fifth (silent CI wait) was resolved via D3. Confirm all five belong in the plan.\n\n```\nFIRST-TIME DEVELOPER REPORT\n============================\nPersona: ML engineer, Python daily driver\nAttempting: EvalKit 2.0.0b1 getting started\n\nT+0:00 Install succeeds. EVALKIT_API_KEY set. README says\n 'python examples/first_eval.py'. I run it.\nT+0:10 FileNotFoundError. No such file. I reread README.\n Nowhere does it warn the file is missing.\nT+0:45 Found 'python -m evalkit.demo' lower in README.\n I run it. Cursor blinks. Nothing.\nT+1:00 Still nothing. Is it hung? No spinner, no output.\nT+3:00 Still nothing. I google 'evalkit demo hanging'.\nT+5:00 Results print. It worked. 5-minute wait not documented\n anywhere in the getting-started path.\nT+5:30 I try to write a real eval. I type run_eval(...) then\n decide to use run_batch instead. Wrong arg order.\n Wrong results. No error.\nT+8:00 I upgrade from v1. My production Client.evaluate() call\n throws AttributeError. I pin back to v1.\nTotal: 7+ min TTHW, 1 AttributeError in prod, 1 silent\n wrong result. 0 actionable errors shown.\n```\n\nRecommendation: A \u2014 all five issues are already agreed (D3\u2013D8); confirm and move to scoring.\n\nNote: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) Address all five issues as agreed in D3\u2013D8 (recommended)\n \u2714 All five decisions are already made; this confirms the plan scope is complete\n \u2714 Together they get TTHW from 7+ minutes to under 2 minutes and eliminate every zero-information error\n \u274c Adds scope to the beta release; each fix is small but requires SDK changes and changelog entries\nB) Let me adjust priorities \u2014 some of these are lower priority than others\n \u2714 Allows de-prioritizing fixes that conflict with release timeline or architecture constraints\n \u274c Means re-opening some of D3\u2013D8; the confusion log shows each issue is a real developer hit\n\nNet: These five issues together turn a 7-minute frustrating onboarding into a sub-2-minute magical moment. Every one is small. Together they are the release.\n\n<gstack-qid:plan-devex-review-confusion-report>": "A) Address all five issues as agreed in D3\u2013D8 (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:03:29.271Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01UCDASxmA9i9j2g6zWjw4hs",
|
||||
"questions": [
|
||||
{
|
||||
"question": "gstack can search learnings from your other projects on this machine to find patterns that might apply here. This stays local (no data leaves your machine). Recommended for solo developers. Skip if you work on multiple client codebases where cross-contamination would be a concern.\n\n<gstack-qid:cross-project-learnings>",
|
||||
"header": "Cross-project",
|
||||
"options": [
|
||||
{
|
||||
"label": "Enable cross-project learnings (Recommended)",
|
||||
"description": "Search for patterns from other projects on this machine when running reviews."
|
||||
},
|
||||
{
|
||||
"label": "Keep learnings project-scoped only",
|
||||
"description": "Limit learnings search to this project only."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"gstack can search learnings from your other projects on this machine to find patterns that might apply here. This stays local (no data leaves your machine). Recommended for solo developers. Skip if you work on multiple client codebases where cross-contamination would be a concern.\n\n<gstack-qid:cross-project-learnings>": "Enable cross-project learnings (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:04:05.425Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_01KeyeMLYxU3SHBPLgFuu8x1",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D10 \u2014 Pass 4: Documentation\n\nFinding: demo output is unspecified \u2014 developer has no reference point for 'success'\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The README says to run `python -m evalkit.demo` but shows no example output. After a 5-minute wait (or after adding the skip flag), the developer sees... something. But they don't know if they should see 10 scores, 100 scores, a single number, or a JSON blob. Without a reference, 'did this work?' requires guesswork. Stripe's docs show you the exact JSON you'll receive. EvalKit's README shows you nothing.\n\nStakes if we pick wrong: A developer who gets output but doesn't recognize it as success may assume it failed and re-run, open an issue, or abandon. This is especially bad for a 5-minute wait \u2014 after sitting through the CI block, they need instant confirmation that it worked.\n\nRecommendation: A \u2014 add expected output to the README. This is a one-paragraph doc change, zero code required, and it makes the first success moment unambiguous.\n\nCompleteness: A=10/10, B=7/10, C=3/10.\n\nPros / cons:\nA) Add to plan: include sample demo output in README under 'Getting Started' (recommended)\n \u2714 Developer immediately knows what success looks like; 'it worked!' moment is unambiguous\n \u2714 Zero code change; one paragraph in README; takes 10 minutes to write\n \u274c Sample output may drift if demo data changes (low risk: sample_responses.json is bundled and stable)\nB) Add as a TODO for post-beta docs polish\n \u2714 Defers the work without blocking the beta\n \u274c The demo is THE magical moment for this release; leaving its output undocumented weakens the whole TTHW fix\nC) Skip \u2014 developer will recognize success when they see scores\n \u2714 Zero effort\n \u274c ML engineers expect scores to be domain-specific; without a reference, 'are these scores correct?' is unanswerable\n\nNet: The getting-started flow now ends with a demo run. If the output isn't documented, the developer's first success moment is ambiguous. A 10-second read of sample output turns ambiguity into confidence.\n\n<gstack-qid:plan-devex-review-todo-demo-output>",
|
||||
"header": "Demo output doc",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Add to plan: include sample demo output in README (Recommended)",
|
||||
"description": "Show what success looks like. One paragraph, zero code change, takes 10 minutes."
|
||||
},
|
||||
{
|
||||
"label": "B) Add as a TODO for post-beta docs polish",
|
||||
"description": "Defer; not blocking the beta, but weakens the TTHW fix."
|
||||
},
|
||||
{
|
||||
"label": "C) Skip \u2014 developer will recognize success when they see scores",
|
||||
"description": "Zero effort. Accepts ambiguous success moment."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D10 \u2014 Pass 4: Documentation\n\nFinding: demo output is unspecified \u2014 developer has no reference point for 'success'\n\nProject/branch: EvalKit SDK 2.0.0b1 / main\n\nELI10: The README says to run `python -m evalkit.demo` but shows no example output. After a 5-minute wait (or after adding the skip flag), the developer sees... something. But they don't know if they should see 10 scores, 100 scores, a single number, or a JSON blob. Without a reference, 'did this work?' requires guesswork. Stripe's docs show you the exact JSON you'll receive. EvalKit's README shows you nothing.\n\nStakes if we pick wrong: A developer who gets output but doesn't recognize it as success may assume it failed and re-run, open an issue, or abandon. This is especially bad for a 5-minute wait \u2014 after sitting through the CI block, they need instant confirmation that it worked.\n\nRecommendation: A \u2014 add expected output to the README. This is a one-paragraph doc change, zero code required, and it makes the first success moment unambiguous.\n\nCompleteness: A=10/10, B=7/10, C=3/10.\n\nPros / cons:\nA) Add to plan: include sample demo output in README under 'Getting Started' (recommended)\n \u2714 Developer immediately knows what success looks like; 'it worked!' moment is unambiguous\n \u2714 Zero code change; one paragraph in README; takes 10 minutes to write\n \u274c Sample output may drift if demo data changes (low risk: sample_responses.json is bundled and stable)\nB) Add as a TODO for post-beta docs polish\n \u2714 Defers the work without blocking the beta\n \u274c The demo is THE magical moment for this release; leaving its output undocumented weakens the whole TTHW fix\nC) Skip \u2014 developer will recognize success when they see scores\n \u2714 Zero effort\n \u274c ML engineers expect scores to be domain-specific; without a reference, 'are these scores correct?' is unanswerable\n\nNet: The getting-started flow now ends with a demo run. If the output isn't documented, the developer's first success moment is ambiguous. A 10-second read of sample output turns ambiguity into confidence.\n\n<gstack-qid:plan-devex-review-todo-demo-output>": "A) Add to plan: include sample demo output in README (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:05:45.958Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "0ce28a08-89c8-4cf6-84ad-5bbbc9c2aacd",
|
||||
"toolUseId": "toolu_011NdN2KRZrdiubotAmj4Ehk",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D11 \u2014 What next?\n\nProject/branch: EvalKit SDK 2.0.0b1 DX review complete / main\n\nELI10: The DX review is done. Eight tasks identified, all with clear files and verify steps. The --skip-ci flag and run_batch arg-order fix both touch client.py and have breaking-change implications \u2014 an eng review should validate both before implementation starts.\n\nRecommendation: A \u2014 run /plan-eng-review next. DX review found two API-level changes (T5 arg order, T1 skip-ci flag) that need architecture validation.\n\nNote: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) Run /plan-eng-review next (recommended)\n \u2714 Validates the --skip-ci flag design and the run_batch arg-order change before anyone starts coding\n \u2714 Required gate for shipping; DX review surfaced issues with architecture implications\n \u274c Adds ~20 minutes before implementation begins\nB) Ready to implement \u2014 run /devex-review after shipping\n \u2714 Start coding immediately with the 8 tasks as the spec\n \u274c Skips the required eng review gate; architectural risks in T1 and T5 are unvalidated\nC) Skip \u2014 I'll handle next steps manually\n \u2714 Full control over what happens next\n \u274c No gate between DX findings and implementation\n\nNet: Eight P1 tasks ready. T1 and T5 have breaking-change implications that warrant a quick eng review before implementation.\n\n<gstack-qid:plan-devex-review-next-steps>",
|
||||
"header": "Next steps",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Run /plan-eng-review next (Recommended)",
|
||||
"description": "Validates --skip-ci flag design and run_batch arg-order breaking change before coding starts."
|
||||
},
|
||||
{
|
||||
"label": "B) Ready to implement \u2014 run /devex-review after shipping",
|
||||
"description": "Start coding from the 8-task spec. Boomerang with /devex-review to verify TTHW target was met."
|
||||
},
|
||||
{
|
||||
"label": "C) Skip \u2014 I'll handle next steps manually",
|
||||
"description": "Full control over what happens next."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D11 \u2014 What next?\n\nProject/branch: EvalKit SDK 2.0.0b1 DX review complete / main\n\nELI10: The DX review is done. Eight tasks identified, all with clear files and verify steps. The --skip-ci flag and run_batch arg-order fix both touch client.py and have breaking-change implications \u2014 an eng review should validate both before implementation starts.\n\nRecommendation: A \u2014 run /plan-eng-review next. DX review found two API-level changes (T5 arg order, T1 skip-ci flag) that need architecture validation.\n\nNote: options differ in kind, not coverage \u2014 no completeness score.\n\nPros / cons:\nA) Run /plan-eng-review next (recommended)\n \u2714 Validates the --skip-ci flag design and the run_batch arg-order change before anyone starts coding\n \u2714 Required gate for shipping; DX review surfaced issues with architecture implications\n \u274c Adds ~20 minutes before implementation begins\nB) Ready to implement \u2014 run /devex-review after shipping\n \u2714 Start coding immediately with the 8 tasks as the spec\n \u274c Skips the required eng review gate; architectural risks in T1 and T5 are unvalidated\nC) Skip \u2014 I'll handle next steps manually\n \u2714 Full control over what happens next\n \u274c No gate between DX findings and implementation\n\nNet: Eight P1 tasks ready. T1 and T5 have breaking-change implications that warrant a quick eng review before implementation.\n\n<gstack-qid:plan-devex-review-next-steps>": "A) Run /plan-eng-review next (Recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-09T02:10:08.919Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
# Settings Page UI Redesign — Design Review
|
||||
|
||||
## Context
|
||||
|
||||
Reviewing the Settings Page UI Redesign plan against the checked-in DESIGN.md.
|
||||
The plan documents existing behavior in detail and calls out 5 implementation gaps
|
||||
explicitly. This review evaluates all 7 design dimensions, confirms each flagged gap,
|
||||
identifies 2 additional gap areas, and proposes concrete fixes traceable to DESIGN.md tokens.
|
||||
|
||||
DESIGN.md exists and is comprehensive. All ratings calibrate against it.
|
||||
|
||||
**Initial rating: 5/10** — Behavior spec is thorough. Design token gaps leave the plan
|
||||
incomplete as an implementation guide. A 10/10 plan has every visual decision traceable
|
||||
to a specific token or measurement with no ambiguity for the implementer.
|
||||
|
||||
---
|
||||
|
||||
## Dimension 1: Information Architecture — 8/10
|
||||
|
||||
**Current state.** The plan specifies h1/h2 heading hierarchy, main landmark, fieldsets
|
||||
with aria-labelledby, and a fixed DOM order. Sections are fixed: Profile and Notifications.
|
||||
Default values for new accounts are defined. Read failure state (Retry) is covered.
|
||||
|
||||
**What a 10 looks like.** The h1 description text is specified verbatim. Every string
|
||||
visible in the UI is either specified or explicitly delegated to an existing source.
|
||||
|
||||
**Gap.** The "short description" beneath the h1 is not specified. The plan says the page
|
||||
header "contains the title, a short description" but gives no copy. For an existing form,
|
||||
this string already exists — the plan should confirm it's unchanged or specify the exact text.
|
||||
|
||||
**Fix.** Add one line: "The description text is unchanged from the current form" or
|
||||
provide the verbatim string. No structural change needed.
|
||||
|
||||
**After fix: 9/10**
|
||||
|
||||
---
|
||||
|
||||
## Dimension 2: Visual Hierarchy — 5/10
|
||||
|
||||
**Current state.** Four buttons in the header action group share the same size, weight,
|
||||
and color. No visual signal tells the user which is primary. InlineStatus position
|
||||
(between actions and form) is correct.
|
||||
|
||||
**What a 10 looks like.** The user's eye lands on Save first without hesitation. The three
|
||||
secondary actions read as secondary. Section headings outweigh field labels. Every
|
||||
element's visual weight matches its semantic importance.
|
||||
|
||||
**Gap (flagged in plan).** Save button must be visually primary. DESIGN.md specifies:
|
||||
Save = #1d4ed8 fill with white text. Reset, Cancel, Export = neutral ghost buttons.
|
||||
The confirmation dialog explains destructive intent for Reset; visual weight reduction
|
||||
there is safe.
|
||||
|
||||
**Additional gap.** Ghost button appearance is specified by name only — "neutral ghost
|
||||
buttons" — with no color tokens for border, text, or hover state. An implementer must
|
||||
guess. This is a common source of inconsistency between developers.
|
||||
|
||||
**Fixes.**
|
||||
|
||||
1. Apply primary style to Save: background-color #1d4ed8, color #ffffff. This is
|
||||
the only filled button on the page.
|
||||
2. Specify ghost style for Reset/Cancel/Export: border 1px solid #d1d5db,
|
||||
color #374151, hover background rgba(0,0,0,0.04). All pairs exceed 4.5:1 on white.
|
||||
3. Add a note: "Export is a ghost button, not a link, because it performs an action
|
||||
(download) rather than navigating."
|
||||
|
||||
**After fixes: 9/10**
|
||||
|
||||
---
|
||||
|
||||
## Dimension 3: Typography — 5/10
|
||||
|
||||
**Current state.** Three font sizes appear across the form — 14px, 16px, 18px —
|
||||
creating a weak and inconsistent hierarchy. The h1 page title likely uses a fourth
|
||||
size not specified in the gap description.
|
||||
|
||||
**What a 10 looks like.** Two sizes. One for all body copy, labels, and helper text.
|
||||
One for section headings. Font weights explicitly specified for each text role so no
|
||||
implementer must guess between 400, 500, and 600.
|
||||
|
||||
**Gap (flagged in plan).** DESIGN.md specifies 16px for body/labels/helper text and
|
||||
20px for section headings (h2). The implementation introduces 14px and 18px, both
|
||||
outside the approved scale.
|
||||
|
||||
**Additional gap.** Font weights are not specified in the plan or DESIGN.md. The
|
||||
implementer faces: Should h2 headings be 600 or 700? Should labels be 500 or 400?
|
||||
What weight for button text, helper text, status text? These choices produce
|
||||
visibly different results and should not be left to implementation judgment.
|
||||
|
||||
**Fixes.**
|
||||
|
||||
1. Replace all 14px with 16px (likely helper text or secondary labels).
|
||||
2. Replace all 18px with 20px if on section headings, or 16px if on body-role text.
|
||||
3. Add font weight specification to the plan:
|
||||
- h1 (page title): font-weight 700
|
||||
- h2 (section headings): font-weight 600
|
||||
- Form labels: font-weight 500
|
||||
- Helper text, status text, descriptions: font-weight 400
|
||||
- Button text: font-weight 500
|
||||
|
||||
**After fixes: 9/10**
|
||||
|
||||
---
|
||||
|
||||
## Dimension 4: Color / Contrast — 5/10
|
||||
|
||||
**Current state.** The error message uses unspecified red text on a light pink background
|
||||
at approximately 3:1 contrast. WCAG AA requires 4.5:1 for normal text. This fails.
|
||||
|
||||
**What a 10 looks like.** Every color value is a specified hex token. Every text/background
|
||||
pairing has a verified contrast ratio in the plan. No state communicates meaning through
|
||||
color alone (icon and text accompany color).
|
||||
|
||||
**Gap (flagged in plan).** Error colors fail WCAG AA. DESIGN.md specifies #991b1b on
|
||||
#fef2f2 with an icon. Calculated contrast for #991b1b on #fef2f2: approximately 8.9:1
|
||||
(well above AA 4.5:1 and AAA 7:1). The DESIGN.md-specified pair is correct.
|
||||
|
||||
**Additional gaps.**
|
||||
|
||||
1. Ghost button text color (#374151) and border color (#d1d5db) need explicit specification
|
||||
(also noted in Dimension 2). Both must be verified against white background: #374151 on
|
||||
white is approximately 8.0:1; #d1d5db is decorative border, not text, so WCAG AA does
|
||||
not apply but Non-text contrast (3:1) does — this should be checked.
|
||||
2. Disabled Save button color during pending state is not specified. A disabled style
|
||||
must not rely solely on low opacity (fails Non-text contrast at thin opacities) and
|
||||
must have an explicit aria-disabled or disabled attribute.
|
||||
3. Error icon color is not specified — it should match #991b1b to avoid a two-color error
|
||||
signal that fragments the semantic unit.
|
||||
|
||||
**Fixes.**
|
||||
|
||||
1. Error text: #991b1b. Error surface: #fef2f2. Error icon: #991b1b. All three explicitly
|
||||
stated in the plan's implementation tasks.
|
||||
2. Ghost button: border #d1d5db (contrast against white: ~1.6:1 — confirm this is a
|
||||
decorative border and not the only focus indicator; the 2px #1d4ed8 focus ring covers
|
||||
keyboard users). Text: #374151.
|
||||
3. Disabled Save during pending: use aria-disabled=true on the button element. Visual
|
||||
appearance: background #93c5fd, text #1e40af (or the existing component's disabled
|
||||
token if one exists). Do not rely on opacity alone.
|
||||
|
||||
**After fixes: 9/10**
|
||||
|
||||
---
|
||||
|
||||
## Dimension 5: Spacing / Rhythm — 6/10
|
||||
|
||||
**Current state.** Three spacing values appear without consistent assignment rules:
|
||||
24px in some places, 32px in others, 16px in a third. Some placements are correct per
|
||||
DESIGN.md (24px field groups, 32px sections), but the appearance of a third value
|
||||
suggests drift from the 8px base scale.
|
||||
|
||||
**What a 10 looks like.** Every spacing value is a multiple of 8px. The implementer
|
||||
reads the spec and knows exactly which spacing applies where without inference.
|
||||
No two sections of the form use different gap values for the same semantic relationship.
|
||||
|
||||
**Gap (flagged in plan).** DESIGN.md specifies the 8px base scale:
|
||||
sections = 32px, field groups = 24px, label-to-input = 8px. The rogue 16px value
|
||||
likely appears where 8px (label-to-input) is specified, or in an un-spec'd relationship
|
||||
(e.g., action row to InlineStatus) where the implementer defaulted to their judgment.
|
||||
|
||||
**Additional gap.** Several spacing relationships are unspecified in the plan and DESIGN.md:
|
||||
- Gap between h1 block (title + description) and action row
|
||||
- Gap between action row and InlineStatus
|
||||
- Gap between InlineStatus and first fieldset
|
||||
- Vertical padding inside fieldsets (around the group of fields)
|
||||
- Gap between a field label and its helper text
|
||||
- Gap between a field and its inline error message
|
||||
|
||||
**Fixes.**
|
||||
|
||||
1. Audit existing implementation for any gap value not on the 8px scale; replace with
|
||||
the nearest 8px multiple.
|
||||
2. Add the following to the plan's spacing spec (all on the 8px scale):
|
||||
- h1 block → action row: 24px
|
||||
- Action row → InlineStatus: 16px
|
||||
- InlineStatus → first fieldset: 24px
|
||||
- Inside fieldset, vertical padding top/bottom: 24px
|
||||
- Field label → helper text: 4px
|
||||
- Field → inline error message: 4px
|
||||
3. Label-to-input remains 8px per DESIGN.md.
|
||||
|
||||
**After fixes: 8/10** — A 10/10 would enumerate dialog internal spacing too; leaving
|
||||
that to the existing ConfirmationDialog component's own spec is an acceptable delegation.
|
||||
|
||||
---
|
||||
|
||||
## Dimension 6: Motion / Feedback States — 5/10
|
||||
|
||||
**Current state.** The Save action takes 2-5 seconds with no visual feedback. The page
|
||||
appears frozen during the operation. Users cannot distinguish between a slow save and a
|
||||
crashed interface.
|
||||
|
||||
**What a 10 looks like.** Every async operation has a defined pending state. Every state
|
||||
transition is perceptible to all users (motion and non-motion). The minimum spinner display
|
||||
time prevents flash. Reduced-motion users receive equivalent status information.
|
||||
|
||||
**Gap (flagged in plan).** DESIGN.md specifies: inline spinner beside "Saving…" text
|
||||
inside the disabled Save button, aria-busy=true, with reduced-motion support.
|
||||
|
||||
**Additional gaps.**
|
||||
|
||||
1. Minimum spinner display time is not specified. For saves completing in under 300ms,
|
||||
the spinner flashes briefly and vanishes — perceptually worse than no spinner. A floor
|
||||
of 500ms prevents the flash while still feeling responsive.
|
||||
2. Dialog open/close transition is not specified. The plan references the existing
|
||||
ConfirmationDialog component. If that component has no animation, state it explicitly
|
||||
so the implementer does not add one ad hoc.
|
||||
3. Error state appearance transition is not specified. Does the error summary animate in
|
||||
or appear immediately? Immediate appearance is correct (no motion delay for errors).
|
||||
|
||||
**Fixes.**
|
||||
|
||||
1. Save pending state:
|
||||
- Disable Save button, set aria-busy=true on the button element.
|
||||
- Show inline spinner (16px, #ffffff, rotating) beside "Saving…" text.
|
||||
- Reduced-motion: omit spinner, show "Saving…" text with a static icon or no icon.
|
||||
- Enforce minimum display time of 500ms: even if the server responds in 100ms,
|
||||
hold the pending state visible for 500ms before transitioning to success.
|
||||
2. Dialog transitions: use existing ConfirmationDialog component defaults. No new
|
||||
animation. Explicitly note this in the implementation tasks.
|
||||
3. Error appearance: immediate, no transition. Explicitly note this.
|
||||
4. Export feedback: see Dimension 7.
|
||||
|
||||
**After fixes: 8/10**
|
||||
|
||||
---
|
||||
|
||||
## Dimension 7: Interaction States & Accessibility — 7/10
|
||||
|
||||
**Current state.** The plan covers a wide range of states: initial load skeleton, new
|
||||
account defaults, read failure (Retry), editing (dirty state via "Unsaved changes"),
|
||||
pending save (gap), save success ("Saved at HH:mm"), save failure (retain edits + error),
|
||||
reset flow (confirmation), cancel flow (confirmation), and export. Accessibility is strong:
|
||||
heading hierarchy, ARIA roles, focus management, 44px targets, visible focus rings,
|
||||
dialog focus trap, and Escape key support.
|
||||
|
||||
**What a 10 looks like.** Every user action has a specified system response. No state is
|
||||
left for the implementer to decide. Validation timing is explicit. All async operations
|
||||
have defined pending states. Edge cases (47-char names, zero-length inputs, duplicate
|
||||
email) are either handled or explicitly deferred.
|
||||
|
||||
**Gap 1: Export pending state is missing.**
|
||||
The plan specifies Export failure and retry but not what happens while the JSON download
|
||||
is being prepared. Is the button disabled? Is there aria-busy? Is there a visual indicator?
|
||||
For a synchronous JSON export (common for client-side serialization), this is instantaneous.
|
||||
If that's the case, state it explicitly — the implementer should not add unnecessary loading
|
||||
UI for a <10ms operation.
|
||||
|
||||
**Gap 2: Validation timing is not specified.**
|
||||
The plan mentions field validation and aria-describedby for errors, and that Save is
|
||||
atomic. But when does validation fire? On blur? On submit? On keystroke? Each produces
|
||||
a different user experience. On-submit-only is consistent with the atomic save pattern
|
||||
and avoids premature errors before the user finishes typing.
|
||||
|
||||
**Gap 3: Field character limits are not specified.**
|
||||
Display name and Email fields have no specified maximum length. If the server enforces
|
||||
a limit (likely), the form should either show a character counter near the limit or
|
||||
surface a clear, specific validation error when exceeded. "Too long" is not specific;
|
||||
"Display name must be 100 characters or fewer (currently 143)" is.
|
||||
|
||||
**Fixes.**
|
||||
|
||||
1. Export pending: "Export is a synchronous client-side JSON serialization. The button
|
||||
disables for the duration of the download prompt (browser-native, < 50ms). No spinner
|
||||
or aria-busy needed. On failure (serialization error), show error in the inline status
|
||||
area per the existing error/retry pattern."
|
||||
2. Validation timing: "Validate on submit only. After a failed submit, field-level errors
|
||||
appear beside each invalid field and in the linked error summary. Editing a field after
|
||||
an error clears that field's error immediately (on input event). Re-validation runs on
|
||||
the next submit attempt."
|
||||
3. Field limits: "Display name: max 100 characters. Email: standard email format validation,
|
||||
max 254 characters (RFC 5321). Both validated server-side; client validation is
|
||||
progressive enhancement only."
|
||||
|
||||
**After fixes: 9/10**
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
| Dimension | Before | After | Critical Fix |
|
||||
|---|---|---|---|
|
||||
| 1. Information Architecture | 8/10 | 9/10 | Confirm h1 description copy |
|
||||
| 2. Visual Hierarchy | 5/10 | 9/10 | Apply primary style to Save; specify ghost tokens |
|
||||
| 3. Typography | 5/10 | 9/10 | Remove 14px/18px; specify font weights |
|
||||
| 4. Color / Contrast | 5/10 | 9/10 | Apply DESIGN.md error colors; specify disabled state |
|
||||
| 5. Spacing / Rhythm | 6/10 | 8/10 | Apply 8px scale; enumerate unspecified gaps |
|
||||
| 6. Motion | 5/10 | 8/10 | Save spinner with 500ms minimum; specify others |
|
||||
| 7. Interaction States | 7/10 | 9/10 | Export pending; validation timing; field limits |
|
||||
|
||||
**Blocking before implementation:** Dimensions 2, 3, 4 (design token gaps will produce
|
||||
inconsistent implementations if unresolved).
|
||||
|
||||
**Resolve before coding starts:** Dimensions 5, 6, 7 (behavioral ambiguities will surface
|
||||
as bugs or review findings if unresolved).
|
||||
|
||||
**Minor / low-risk:** Dimension 1 (copy confirmation).
|
||||
|
||||
---
|
||||
|
||||
## GSTACK REVIEW REPORT
|
||||
|
||||
| Run | Status | Findings |
|
||||
|---|---|---|
|
||||
| plan-design-review | complete | 7 dimensions reviewed, 13 findings across all dimensions |
|
||||
| Outside voices | skipped | text-only mode per user request |
|
||||
| Mockups | skipped | text-only mode per user request |
|
||||
|
||||
**VERDICT:** NEEDS_REVISION — 5 of 7 dimensions rated below 7/10. Three dimensions (Visual Hierarchy, Typography, Color) have gaps that will produce inconsistent implementations. All fixes trace directly to DESIGN.md tokens; no design decisions require new choices. Apply the DESIGN.md-specified tokens and add the specificity items to the implementation plan.
|
||||
|
||||
**UNRESOLVED DECISIONS:**
|
||||
- D1: Confirm h1 description copy is unchanged from current form, or specify verbatim string
|
||||
- D2: Specify ghost button border/text/hover tokens (border #d1d5db, text #374151 recommended)
|
||||
- D2: Confirm disabled Save visual tokens during pending state
|
||||
- D3: Add font weight spec to plan (h1:700, h2:600, label:500, helper:400, button:500 recommended)
|
||||
- D5: Enumerate unspecified spacing relationships (h1→actions, actions→status, status→fieldset)
|
||||
- D6: Confirm 500ms minimum spinner display time; confirm no dialog transition animation
|
||||
- D7: Confirm Export is synchronous (no pending state needed); specify validation fires on submit only; specify field character limits
|
||||
@@ -1,4 +1,4 @@
|
||||
import { capturePlanCountQuestion, planCountPrerequisitePick, planCountQuestionInput, type AskUserQuestionFingerprint } from './claude-pty-runner';
|
||||
import { capturePlanCountQuestion, parseNumberedOptions, planCountPrerequisitePick, planCountQuestionInput, planCountSubmissionInput, type AskUserQuestionFingerprint } from './claude-pty-runner';
|
||||
|
||||
import type { NativePlanQuestionCall } from './plan-count-transcript';
|
||||
|
||||
@@ -49,7 +49,7 @@ function clippedRoutingTitle(visible: string, question: AskUserQuestionFingerpri
|
||||
return title;
|
||||
}
|
||||
|
||||
/** Fail only a complete current setup panel; absence or stale/partial metadata is not failure. */
|
||||
/** Identify a complete current setup panel; absence or stale/partial metadata is insufficient. */
|
||||
function completeSetupOptions(visible: string, pending?: NativePlanQuestionCall): Array<{ index: number; label: string }> | null {
|
||||
if (!activeSetupPanel(visible)) return null;
|
||||
const lines = visible.replace(/\r+\n?/g, '\n').trimEnd().split('\n');
|
||||
@@ -87,9 +87,131 @@ function unsupportedSetup(visible: string, question: AskUserQuestionFingerprint,
|
||||
identitySource: pending ? 'native-bound' : 'current-native-panel' } : { kind: 'waiting' };
|
||||
}
|
||||
|
||||
/** Pure routing policy; native packet validation still requires every displayed identity. */
|
||||
function routingSetupActions(question: AskUserQuestionFingerprint, allowTemporarySkip: boolean) {
|
||||
const primary = question.promptSnippet.replace(/^(?:Routing\s*rules|CLAUDE\.md)\s*/i, '').split('?', 1)[0]!;
|
||||
const prompt = primary.replace(/\s+/g, '');
|
||||
const options = question.options.map(option => ({
|
||||
index: option.index,
|
||||
title: option.label.split(/[│┌\r\n]/, 1)[0]!.replace(/\s+/g, ''),
|
||||
}));
|
||||
const add = options.filter(option => /^Add(?:routingrules(?:toCLAUDE\.md)?|toCLAUDE\.md)(?:\(Recommended\))?$/i.test(option.title));
|
||||
// Match the declined setup action, not every English label separately:
|
||||
// No thanks/Skip may stand alone or opt into manual invocation. A manual
|
||||
// migration, deletion, or unrelated workflow is not the opposed action.
|
||||
// "Only" limits the same manual action; it does not add a second action.
|
||||
// Use one whole-label grammar with and without a courtesy/Skip prefix.
|
||||
const manualAction = /^(?:manual(?:invocation)?|(?:I['’]ll)?invoke(?:skills)?manually)(?:[-–—]?only)?$/i;
|
||||
// A temporary Skip is the same opposed setup action only on an intact
|
||||
// two-choice panel. Its description may corroborate manual invocation;
|
||||
// the routing premise and unique Add action below establish its scope.
|
||||
const decline = options.filter(option => {
|
||||
const title = option.title.replace(/\(Recommended\)$/i, '');
|
||||
if (/^Skipfornow$/i.test(title)) return allowTemporarySkip;
|
||||
// The action can stand alone or follow a short courtesy ('No thanks').
|
||||
// Cursor redraws can damage that courtesy while leaving 'invoke skills
|
||||
// manually' intact. Match the complete action, not the spelling of No;
|
||||
// arbitrary preceding instructions and extra trailing actions still fail.
|
||||
const manual = title.replace(/^[a-z]{0,3}thanks[,—–-]/i, '');
|
||||
if (manualAction.test(manual)) return true;
|
||||
const prefix = /^(?:Nothanks|Skip)(?:[,—–-])?/i.exec(title);
|
||||
if (!prefix) return false;
|
||||
// 'No thanks' can be followed by the same explicit Skip action. Strip
|
||||
// that decline verb before checking any optional manual-invocation text.
|
||||
const action = title.slice(prefix[0].length).replace(/^skip(?:[,—–-])?/i, '');
|
||||
return action === '' || manualAction.test(action);
|
||||
});
|
||||
const routingId = /<gstack-qid:routing-injection>/i.test(question.promptSnippet);
|
||||
const routingPremise = /gstack/i.test(prompt) && /CLAUDE\.md/i.test(prompt) && /skillroutingrules/i.test(prompt);
|
||||
// A qid can replace the longer premise, but cannot override a question
|
||||
// about a different target. The Add action and question must agree on
|
||||
// project setup rather than a product routing or taste decision.
|
||||
const claudeTarget = /CLAUDE\.md/i.test(prompt);
|
||||
const quotedPremise = /\b(?:plan|spec|document)\s+(?:quotes?|cites?|references?)\b/i.test(primary);
|
||||
if (!claudeTarget || quotedPremise || (!routingId && !routingPremise)) return null;
|
||||
return { add, decline };
|
||||
}
|
||||
|
||||
/** Answer only the known pair of setup offers, using the actual native active tab. */
|
||||
function setupPacketDecision(visible: string, seen: ReadonlySet<string>, pending: NativePlanQuestionCall): AutoplanSetupDecision {
|
||||
const waiting: AutoplanSetupDecision = { kind: 'waiting' };
|
||||
// Validate all questions before touching any tab: a setup question cannot
|
||||
// lend its policy to an adjacent finding, taste decision or checkbox.
|
||||
if (pending.questions.length !== 2) return waiting;
|
||||
const policies = pending.questions.map(question => {
|
||||
if (question.options.length !== 2) return null;
|
||||
const ids = [...question.question.matchAll(/<gstack-qid:[a-z0-9-]+>/gi)];
|
||||
if (ids.length > 1 || (question.question.match(/<gstack-qid/gi)?.length ?? 0) !== ids.length) return null;
|
||||
const offerText = question.question.replace(/<gstack-qid:[a-z0-9-]+>/gi, '').trim();
|
||||
const fp: AskUserQuestionFingerprint = { signature: '', observedAtMs: 0, preReview: true,
|
||||
promptSnippet: `${question.header} ${question.question}`,
|
||||
options: question.options.map((option, index) => ({ index: index + 1, label: option.label })) };
|
||||
const routing = routingSetupActions(fp, true);
|
||||
// A packet must contain only setup. Scope the entire question, including
|
||||
// any premise, rather than borrowing the first question mark's identity
|
||||
// while a later sentence asks for an unrelated approval.
|
||||
const routingOffer = /^(?:gstack\s+works\s+best\s+when\s+(?:your|this|the)\s+project['’]s\s+CLAUDE\.md\s+includes\s+skill\s+routing\s+rules\.\s*)?(?:(?:Should|Can)\s+(?:I|gstack)\s+|Would\s+you\s+like\s+(?:me|gstack)\s+to\s+)?Add\s+(?:(?:gstack\s+)?skill\s+routing\s+rules\s+to\s+(?:this\s+project['’]s\s+)?CLAUDE\.md|(?:skill\s+)?routing\s+rules(?:\s+to\s+CLAUDE\.md)?|them)(?:\s+now)?\?$/i.test(offerText);
|
||||
if (routingOffer && routing?.add.length === 1 && routing.decline.length === 1 && routing.add[0]!.index !== routing.decline[0]!.index) {
|
||||
return { kind: 'routing', pick: routing.add[0]!.index };
|
||||
}
|
||||
const prerequisite = planCountPrerequisitePick(fp);
|
||||
const run = question.options.filter(option => /^Run\s*\/office-hours\s*(?:now|first)(?:\s*\(recommended\))?$/i.test(option.label));
|
||||
// Require an actual prerequisite offer, not a product question that
|
||||
// happens to mention the absence of an office-hours design document.
|
||||
const offer = /^No\s+design\s+doc\s+(?:found|exists)(?:\s+for\s+(?:this|the)\s+(?:branch|project))?\.\s*(?:\/office-hours\s+(?:produces|creates|provides)\s+(?:a\s+)?(?:structured\s+)?(?:design\s+doc(?:ument)?|problem\s+statement)(?:,?\s+(?:and\s+)?(?:premise\s+challenge|(?:explored\s+)?alternatives))*(?:\s*[—–-]\s*(?:sharper|better)\s+input\s+for\s+(?:the|this)\s+review)?\.\s*)?(?:Want\s+to\s+|Would\s+you\s+like\s+to\s+)?Run\s+(?:it|\/office-hours)\s+(?:now|first)(?:\s+or\s+proceed\s+with\s+standard\s+review)?\s*\?$/i.test(offerText);
|
||||
return prerequisite !== null && run.length === 1 && offer ? { kind: 'prerequisite', pick: prerequisite } : null;
|
||||
});
|
||||
if (policies.some(policy => !policy) || new Set(policies.map(policy => policy!.kind)).size !== 2) return waiting;
|
||||
|
||||
const text = visible.replace(/\r+\n?/g, '\n').trimEnd();
|
||||
const bars = [...text.matchAll(/^ {0,3}←([^\n]*[☐☒][^\n]*)✔\s*Submit\s*→[\t ]*$/gm)];
|
||||
const footer = /Enter[\t ]+to[\t ]+select[\t ]*·[\t ]*Tab\/Arrow[\t ]+keys[\t ]+to[\t ]+navigate[\t ]*·[\t ]*Esc[\t ]+to[\t ]+cancel$/i;
|
||||
if (bars.length !== 1 || !footer.test(text)) return waiting;
|
||||
const bar = bars[0]!;
|
||||
const tabs = [...bar[1]!.matchAll(/([☐☒])\s*([^☐☒]+)/g)];
|
||||
const compact = (value: string) => value.replace(/\s+/g, '');
|
||||
const introduction = text.slice(0, bar.index).split('\n').findLast(line => !/^[\t ─━-]*$/.test(line)) ?? '';
|
||||
if (/\b(?:example|sample|quot(?:e[sd]?|ed)|template|source)\b[^\n]*:\s*$/i.test(introduction) ||
|
||||
compact(bar[1]!) !== tabs.map(tab => compact(tab[0])).join('') ||
|
||||
tabs.length !== pending.questions.length || tabs.some((tab, index) => compact(tab[2]!) !== compact(pending.questions[index]!.header)) ||
|
||||
/(?:^|\n)[^\n]*[1-9]\.\s*\[[ ✓✔xX]\]/.test(text)) return waiting;
|
||||
// Project only this actual pane's decoration for the existing full-panel
|
||||
// validator. The question, labels and native identity remain unchanged.
|
||||
const project = (header: string) => (text.slice(0, bar.index) + '☐ ' + header +
|
||||
text.slice(bar.index + bar[0].length).replace(footer, 'Enter to select · ↑/↓ to navigate · Esc to cancel'))
|
||||
.replace(/(^|\n)[\t ]*[│┃][\t ]?/g, '$1');
|
||||
if (!activeSetupPanel(project('Setup packet'))) return waiting;
|
||||
const packetKey = 'autoplan-setup-packet:' + JSON.stringify({sessionId:pending.sessionId,toolUseId:pending.toolUseId,questions:pending.questions});
|
||||
const choiceKey = (index: number) => `${packetKey}:choice:${index}:${policies[index]!.pick}`;
|
||||
const submitKey = packetKey + ':submit';
|
||||
if (planCountSubmissionInput(text) === '\r') {
|
||||
// Checked tabs are corroboration. Only choices this caller actually
|
||||
// sent for this same native packet can authorize its final submission.
|
||||
const options = parseNumberedOptions(text);
|
||||
if (!/Ready\s+to\s+submit\s+your\s+answers\?\s*❯\s*1\./.test(text) ||
|
||||
options.length !== 2 || options[0]?.index !== 1 || options[0]?.label !== 'Submit answers' ||
|
||||
options[1]?.index !== 2 || options[1]?.label !== 'Cancel' || seen.has(submitKey) ||
|
||||
tabs.some((tab, index) => tab[1] !== '☒' || !seen.has(choiceKey(index)))) return waiting;
|
||||
return { kind: 'input', input: '\r', signatures: [submitKey] };
|
||||
}
|
||||
|
||||
const captured = new Set(seen);
|
||||
const fp = capturePlanCountQuestion(text, captured, 0, true, pending);
|
||||
const index = fp?.nativeQuestionIndex;
|
||||
if (!fp || fp.nativeCall !== pending || index === undefined || tabs[index]?.[1] !== '☐' || seen.has(choiceKey(index))) return waiting;
|
||||
const question = pending.questions[index]!;
|
||||
if (!completeSetupOptions(project(question.header), { ...pending, questions: [question] })) return waiting;
|
||||
return { kind: 'input', input: planCountQuestionInput(text, fp, policies[index]!.pick),
|
||||
signatures: [...captured].filter(signature => !seen.has(signature)).concat(choiceKey(index)) };
|
||||
}
|
||||
|
||||
/** Pure classification: only the caller that sends input commits returned identities. */
|
||||
export function autoplanSetupDecision(visible: string, seen: ReadonlySet<string>, pending?: NativePlanQuestionCall): AutoplanSetupDecision {
|
||||
if (pending && (pending.answered || pending.failed || pending.questions.length !== 1 || pending.questions[0]?.multiSelect)) return { kind: 'waiting' };
|
||||
if (pending && (pending.answered || pending.failed || !pending.questions.length || pending.questions.some(question => question.multiSelect))) return { kind: 'waiting' };
|
||||
if (pending && pending.questions.length > 1) return setupPacketDecision(visible, seen, pending);
|
||||
// A visible packet without its complete native metadata cannot prove that
|
||||
// its other tabs are setup. Wait for persistence instead of guessing.
|
||||
if (/←[^\r\n]*[☐☒][^\r\n]*✔\s*Submit\s*→|Enter\s*to\s*select\s*·\s*Tab\/Arrow\s*keys\s*to\s*navigate/.test(visible)) return { kind: 'waiting' };
|
||||
// Box borders are terminal decoration, not part of an untagged native
|
||||
// question's wrapped text. Keep its full content for identity matching.
|
||||
const display = visible.replace(/(^|[\r\n])[\t ]*[│┃][\t ]?/g, '$1');
|
||||
@@ -139,42 +261,11 @@ export function autoplanSetupDecision(visible: string, seen: ReadonlySet<string>
|
||||
// asked; an exact "Add them now?" sentence is not a stable interface.
|
||||
// Keep the actual question/premise separate from its header and later ELI10
|
||||
// prose, which may mention CLAUDE.md even on an unrelated question.
|
||||
const primary = question.promptSnippet.replace(/^(?:Routing\s*rules|CLAUDE\.md)\s*/i, '').split('?', 1)[0]!;
|
||||
const prompt = primary.replace(/\s+/g, '');
|
||||
const options = question.options.map(option => ({
|
||||
index: option.index,
|
||||
title: option.label.split(/[│┌\r\n]/, 1)[0]!.replace(/\s+/g, ''),
|
||||
}));
|
||||
const add = options.filter(option => /^Add(?:routingrules(?:toCLAUDE\.md)?|toCLAUDE\.md)(?:\(Recommended\))?$/i.test(option.title));
|
||||
// Match the declined setup action, not every English label separately:
|
||||
// No thanks/Skip may stand alone or opt into manual invocation. A manual
|
||||
// migration, deletion, or unrelated workflow is not the opposed action.
|
||||
// "Only" limits the same manual action; it does not add a second action.
|
||||
// Use one whole-label grammar with and without a courtesy/Skip prefix.
|
||||
const manualAction = /^(?:manual(?:invocation)?|(?:I['’]ll)?invoke(?:skills)?manually)(?:[-–—]?only)?$/i;
|
||||
const decline = options.filter(option => {
|
||||
const title = option.title.replace(/\(Recommended\)$/i, '');
|
||||
// The action can stand alone or follow a short courtesy ('No thanks').
|
||||
// Cursor redraws can damage that courtesy while leaving 'invoke skills
|
||||
// manually' intact. Match the complete action, not the spelling of No;
|
||||
// arbitrary preceding instructions and extra trailing actions still fail.
|
||||
const manual = title.replace(/^[a-z]{0,3}thanks[,—–-]/i, '');
|
||||
if (manualAction.test(manual)) return true;
|
||||
const prefix = /^(?:Nothanks|Skip)(?:[,—–-])?/i.exec(title);
|
||||
if (!prefix) return false;
|
||||
// 'No thanks' can be followed by the same explicit Skip action. Strip
|
||||
// that decline verb before checking any optional manual-invocation text.
|
||||
const action = title.slice(prefix[0].length).replace(/^skip(?:[,—–-])?/i, '');
|
||||
return action === '' || manualAction.test(action);
|
||||
});
|
||||
const routingId = /<gstack-qid:routing-injection>/i.test(question.promptSnippet);
|
||||
const routingPremise = /gstack/i.test(prompt) && /CLAUDE\.md/i.test(prompt) && /skillroutingrules/i.test(prompt);
|
||||
// A qid can replace the longer premise, but cannot override a question
|
||||
// about a different target. The Add action and question must agree on
|
||||
// project setup rather than a product routing or taste decision.
|
||||
const claudeTarget = /CLAUDE\.md/i.test(prompt);
|
||||
const quotedPremise = /\b(?:plan|spec|document)\s+(?:quotes?|cites?|references?)\b/i.test(primary);
|
||||
if (!claudeTarget || quotedPremise || (!routingId && !routingPremise)) return { kind: 'unrelated' };
|
||||
const temporarySkipPanel = question.options.some(option => /^Skip\s*for\s*now(?:\s*\(Recommended\))?$/i.test(option.label))
|
||||
? completeSetupOptions(display, pending) : null;
|
||||
const actions = routingSetupActions(question, temporarySkipPanel?.length === 2);
|
||||
if (!actions) return { kind: 'unrelated' };
|
||||
const { add, decline } = actions;
|
||||
if (pending && (!question.nativeCall || pending.questions.length !== 1 || pending.questions[0]?.multiSelect)) return { kind: 'waiting' };
|
||||
// An intact Add-to-CLAUDE.md action identifies this setup offer even if
|
||||
// its opposed decline is unsupported. A qid or premise alone must not
|
||||
|
||||
@@ -13,6 +13,24 @@ function resolvedCeoRecap(description: string): boolean {
|
||||
return Boolean(clause && !/\b(?:if|unless|until|once|when|after|not|never|some|most|partially|only|of|but|several|few)\b|n['’]t\b/i.test(clause));
|
||||
}
|
||||
|
||||
/** A closed-review declaration plus one direct navigation query, even when its recap follows it. */
|
||||
function closedReviewNavigation(declaration: string, context: string): boolean {
|
||||
const question = declaration.replace(/<gstack-qid:[^>]+>/gi, '');
|
||||
const unfinished = context.replace(/\b(?:no|0)\s+unresolved\s+(?:decisions|gaps|issues|findings)\b/gi, '');
|
||||
// Conditional closure of this review is unfinished work. Sequencing the
|
||||
// next review after implementation does not reopen the completed CEO review.
|
||||
const stateVerb = String.raw`(?:is|are|was|were|becomes?|became|(?:will|would|can|could|may|might)\s+(?:be|become))`;
|
||||
const closure = String.raw`(?:(?:all\s+)?(?:decisions|gaps|issues|findings)\s+(?:${stateVerb}\s+)?resolved|(?:the\s+)?CEO\s+review\s+(?:${stateVerb}\s+)?(?:complete|done|cleared|clean)|the\s+review\s+(?:${stateVerb}\s+)?(?:complete|done|cleared|clean))`;
|
||||
const conditionalClosure = new RegExp(String.raw`\b(?:once|when|after)\b[^.!?]{0,180}\b${closure}\b|\b${closure}\b[^.!?]{0,100}\b(?:once|when|after)\b`, 'i');
|
||||
return /^CEO review (?:is )?(?:complete|done|cleared|clean)[.!](?:\s|$)/i.test(question) &&
|
||||
/(?:^|[.!]\s+)What(?:['’]s)? next\?(?:\s|$)/i.test(question) &&
|
||||
(context.match(/\?/g)?.length ?? 0) === 1 &&
|
||||
!/\b(?:unresolved|outstanding|remaining|pending|if|unless|until)\b|\b(?:gap|issue|finding|decision)s?\s+(?:still\s+)?remains?\b|\bstill\s+open\b/i.test(unfinished) &&
|
||||
!/\bnot\s+(?:all|no|0)\b/i.test(context) &&
|
||||
!conditionalClosure.test(context) &&
|
||||
!/(?:^|[.!?;]\s+|\b(?:proceed to|continue to|should|must|will|need to|can|could|would)\s+)(?:(?:please|first|then|also)\s+)*(?:add|fix|implement|resolve|decide)\b/im.test(context);
|
||||
}
|
||||
|
||||
/** Only the CEO's finished-review menu, never a finding/TODO mentioning another skill. */
|
||||
function manualHandoffIndex(fp: AskUserQuestionFingerprint): number | null {
|
||||
const call = fp.nativeCall;
|
||||
@@ -24,7 +42,7 @@ function manualHandoffIndex(fp: AskUserQuestionFingerprint): number | null {
|
||||
const ids = [...q.question.matchAll(/<gstack-qid:\s*([a-z0-9-]+)\s*>/gi)];
|
||||
if (ids.length > 1 || (q.question.match(/<gstack-qid/gi)?.length ?? 0) !== ids.length) return null;
|
||||
const id = ids[0]?.[1]?.toLowerCase();
|
||||
if (id && !/^(?:plan-ceo-(?:review-)?next-(?:steps?|review)|ceo-next-step-eng-review|ceo-plan-next-steps)$/.test(id)) return null;
|
||||
if (id && !/^(?:plan-ceo-(?:review-)?next-(?:steps?|review)|ceo-review-next-(?:steps?|review)|ceo-next-step-eng-review|ceo-plan-next-steps)$/.test(id)) return null;
|
||||
const declaration = q.question.replace(/^D\s*\d+\s*[—–:-]\s*/i, '')
|
||||
.replace(/^next\s+(?:review|steps?)\s*:\s*/i, '');
|
||||
const gateContext = [q.question, ...q.options.map(option => option.description ?? '')].join('\n');
|
||||
@@ -41,14 +59,10 @@ function manualHandoffIndex(fp: AskUserQuestionFingerprint): number | null {
|
||||
const completion = explicitCompletion || describedCompletion;
|
||||
const requiredEng = /(?:\bEng(?:ineering)?\s+review|\/plan-eng-review)\b[^.!?]{0,180}\brequired(?:\s+shipping)?\s+gate\b/i.test(gateContext) ||
|
||||
/\brequired(?:\s+shipping)?\s+gate\s+is\s+(?:an?\s+)?(?:Eng(?:ineering)?\s+review|\/plan-eng-review)\b/i.test(gateContext);
|
||||
// This native identity still needs an unconditional closed-review heading;
|
||||
// a next-step label cannot conceal a new repair or an unfinished obligation.
|
||||
if (id === 'ceo-plan-next-steps' &&
|
||||
(!/^CEO review (?:is )?complete[.!](?:\s|$)/i.test(declaration) ||
|
||||
!/\bWhat(?:['’]s)? next\?\s*<gstack-qid:ceo-plan-next-steps>\s*$/i.test(declaration) ||
|
||||
(declaration.match(/\?/g)?.length ?? 0) !== 1 ||
|
||||
/\b(?:unresolved|outstanding|remaining|pending|if|unless|until)\b|\b(?:gap|issue|finding|decision)s?\s+(?:still\s+)?remains?\b/i.test(gateContext) ||
|
||||
/(?:^|[.!?;]\s+|\b(?:proceed to|continue to|should|must|will|need to)\s+)(?:(?:please|first|then|also)\s+)*(?:add|fix|implement|resolve|decide)\b/im.test(gateContext))) return null;
|
||||
// These native next-review identities share a closed navigation contract;
|
||||
// the question or a following recap cannot hide a new repair obligation.
|
||||
if (id && /^(?:ceo-plan-next-steps|ceo-review-next-(?:steps?|review))$/.test(id) &&
|
||||
!closedReviewNavigation(declaration, gateContext)) return null;
|
||||
// A qid names the menu; it cannot replace its completed-review declaration
|
||||
// or authorize another fix. The named gate can be explained in a choice.
|
||||
if (!/^next\s+(?:review|steps?)$/i.test(q.header.trim()) || !completion || !requiredEng) return null;
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
isNumberedOptionListVisible,
|
||||
matchesNativePlanQuestion,
|
||||
planCountSubmissionInput,
|
||||
planCountPrerequisitePick,
|
||||
type AskUserQuestionFingerprint,
|
||||
} from './claude-pty-runner';
|
||||
import type { NativePlanQuestionCall, PlanCountTranscript } from './plan-count-transcript';
|
||||
@@ -48,7 +49,7 @@ export function findCeoModeOption(
|
||||
type ModeNavigationAction =
|
||||
| { kind: 'wait' }
|
||||
| { kind: 'permission' | 'submission'; input: string }
|
||||
| { kind: 'question'; question: AskUserQuestionFingerprint }
|
||||
| { kind: 'question'; index: number; question: AskUserQuestionFingerprint }
|
||||
| { kind: 'mode'; index: number; question: AskUserQuestionFingerprint };
|
||||
|
||||
// Permission state follows the navigation session without entering its AUQ
|
||||
@@ -90,7 +91,11 @@ export function nextCeoModeNavigation(
|
||||
const question = capturePlanCountQuestion(visible, seenQuestions, 0, true, pending);
|
||||
if (!question) return { kind: 'wait' };
|
||||
const index = findCeoModeOption(question.options, targetMode);
|
||||
return index === null ? { kind: 'question', question } : { kind: 'mode', index, question };
|
||||
// Preserve the seeded review plan by declining the existing optional
|
||||
// office-hours offer. Other navigation questions keep their first choice.
|
||||
return index === null
|
||||
? { kind: 'question', index: planCountPrerequisitePick(question) ?? 1, question }
|
||||
: { kind: 'mode', index, question };
|
||||
}
|
||||
|
||||
/** Match new assistant prose, never the native mode menu or answer echo. */
|
||||
|
||||
@@ -31,7 +31,7 @@ import { withHermeticSkillRuntime } from './hermetic-skill-runtime';
|
||||
import { createPlanCountFixture } from './plan-count-fixture';
|
||||
import { createPlanCountSnapshotWriter } from './plan-count-artifacts';
|
||||
import { readPlanCountTranscript, unresolvedPlanQuestionCalls, type NativePlanQuestionCall, type PlanCountTranscript } from './plan-count-transcript';
|
||||
import { createPendingExitRecorder, withPendingExit } from './plan-count-pending-exit';
|
||||
import { createPendingExitRecorder, withPendingExit, isCurrentPlanApprovalScreen } from './plan-count-pending-exit';
|
||||
import { trustDialogInput } from './pty-trust-dialog';
|
||||
import { createPtyScreen } from './pty-screen';
|
||||
|
||||
@@ -1641,7 +1641,8 @@ export function hasNativePlanCompletion(
|
||||
return hasCompletePlanReport(expectedPlanPath, Math.max(startedAt, ...answerTimes), finishedAt);
|
||||
}
|
||||
|
||||
function hasCompletePlanReport(expectedPlanPath: string, minimumMtime: number, maximumMtime: number): boolean {
|
||||
function hasCompletePlanReport(expectedPlanPath: string, minimumMtime: number, maximumMtime: number,
|
||||
allowRunHeaderForFailure = false): boolean {
|
||||
if (!path.isAbsolute(expectedPlanPath)) return false;
|
||||
try {
|
||||
const stat = fs.lstatSync(expectedPlanPath);
|
||||
@@ -1673,7 +1674,13 @@ function hasCompletePlanReport(expectedPlanPath: string, minimumMtime: number, m
|
||||
const report = content.slice(content.indexOf('## GSTACK REVIEW REPORT'));
|
||||
// Reject a provisional heading; retain both clean and issues-open reports.
|
||||
const rows = report.split('\n');
|
||||
const table = rows.findIndex(line => /^\|[^\n]*Review[^\n]*\|/.test(line));
|
||||
// Positive completion keeps the canonical Review header. A failure-only
|
||||
// diagnostic may also recognize the Run header found in a native report;
|
||||
// it still requires a complete, fresh caller-owned deliverable.
|
||||
const tableHeader = allowRunHeaderForFailure
|
||||
? /^\|(?:[^\n|]*\|)*[ \t]*(?:Review|Run)[ \t]*\|/
|
||||
: /^\|[^\n]*Review[^\n]*\|/;
|
||||
const table = rows.findIndex(line => tableHeader.test(line));
|
||||
const completeTable = table >= 0 && /^\|[ :|-]+\|$/.test(rows[table + 1] ?? '') &&
|
||||
/^\|.*[a-zA-Z].*\|$/.test(rows[table + 2] ?? '');
|
||||
const decisions = rows.findIndex(line => /^\*\*UNRESOLVED DECISIONS:\*\*$/.test(line));
|
||||
@@ -1684,6 +1691,22 @@ function hasCompletePlanReport(expectedPlanPath: string, minimumMtime: number, m
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
/** A final owned gate with no recorded questions is missing coverage, never success. */
|
||||
export function isQuestionlessNativePlanExit(
|
||||
transcript: PlanCountTranscript, expectedPlanPath: string, startedAt: number, screen: string,
|
||||
): boolean {
|
||||
if (!isCurrentPlanApprovalScreen(screen)) return false;
|
||||
if (transcript.status !== 'ready' || transcript.calls.length) return false;
|
||||
const ready = [...(transcript.planReadyRequests ?? [])]
|
||||
.sort((a, b) => Date.parse(a.timestamp) - Date.parse(b.timestamp)).at(-1);
|
||||
if (!ready || ready.failed || !ready.sessionId || !ready.toolUseId) return false;
|
||||
const at = Date.parse(ready.timestamp);
|
||||
const sessions = new Set([...transcript.assistantMessages.map(m => m.sessionId),
|
||||
...(transcript.planReadyRequests ?? []).map(r => r.sessionId)]);
|
||||
return sessions.size === 1 && Number.isFinite(at) && at >= startedAt && at <= Date.now() &&
|
||||
hasCompletePlanReport(expectedPlanPath, startedAt, at, true);
|
||||
}
|
||||
|
||||
/** Native report completion is independent of the terminal's streamed headings. */
|
||||
export function hasNativePlanTerminal(
|
||||
transcript: PlanCountTranscript,
|
||||
@@ -1757,13 +1780,39 @@ function isCompletedDxHandoff(call: NativePlanQuestionCall): boolean {
|
||||
!Array.isArray(call.unansweredQuestionIndices) || call.unansweredQuestionIndices.length) return false;
|
||||
const q = call.questions[0]!;
|
||||
const header = q.header.trim().replace(/^D\s*\d+\s*(?:[—–:-]\s*)?/i, '');
|
||||
const question = q.question.replace(/^D\s*\d+\s*[—–:-]\s*/i, '');
|
||||
const completionLines: string[] = [];
|
||||
let completionFence: { marker: string; length: number } | undefined;
|
||||
for (const line of question.split('\n')) {
|
||||
const delimiter = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line);
|
||||
if (delimiter) {
|
||||
if (!completionFence) completionFence = { marker: delimiter[1]![0]!, length: delimiter[1]!.length };
|
||||
else if (delimiter[1]![0] === completionFence.marker && delimiter[1]!.length >= completionFence.length && !delimiter[2]!.trim()) completionFence = undefined;
|
||||
} else if (!completionFence && !/^(?: {4}|\t|\s*>)/.test(line)) completionLines.push(line);
|
||||
}
|
||||
const completionProse = completionLines.join('\n');
|
||||
const closedReviewNavigation = /^What(?:['’]s)? next\?\s*\n/i.test(question) &&
|
||||
/(?:^|\n)(?:ELI10:\s*)?(?:The )?DX review (?:is )?(?:done|complete)[.!](?:\s|$)/i.test(completionProse);
|
||||
if (q.multiSelect || !/^next(?:\s+steps?)?$/i.test(header) ||
|
||||
!/^DX review (?:is )?(?:done|complete)[.!]/i.test(q.question.replace(/^D\s*\d+\s*[—–:-]\s*/i, ''))) return false;
|
||||
(!closedReviewNavigation && !/^DX review (?:is )?(?:done|complete)[.!]/i.test(question))) return false;
|
||||
const ids = [...q.question.matchAll(/<gstack-qid:([^>]+)>/gi)];
|
||||
if ((q.question.match(/<gstack-qid/gi)?.length ?? 0) !== ids.length) return false;
|
||||
let resultRecap = false;
|
||||
if (ids.length) {
|
||||
if (ids.length !== 1 || ids[0]![1] !== 'devex-next-steps') return false;
|
||||
if (ids.length !== 1 || ids[0]![1] !== (closedReviewNavigation
|
||||
? 'plan-devex-review-next-steps' : 'devex-next-steps')) return false;
|
||||
if (closedReviewNavigation) {
|
||||
const context = [q.question, ...q.options.map(o => o.description ?? '')].join('\n');
|
||||
const state = '(?:(?:is|are|was|were|becomes?|became)|(?:will|would|can|could|may|might) (?:be|become))';
|
||||
const closure = `(?:the )?(?:DX review (?:${state} )?(?:done|complete)|(?:all )?(?:decisions?|gaps?|issues?|findings?) (?:${state} )?resolved)`;
|
||||
const conditionalClosure = new RegExp(`\\b(?:once|when|after)\\b[^.!?\\n]*${closure}|${closure}[^.!?\\n]*\\b(?:once|when|after)\\b`, 'i');
|
||||
if ((q.question.match(/\?/g)?.length ?? 0) !== 1 ||
|
||||
q.options.some(option => /\?/.test(option.description ?? '')) ||
|
||||
!/\brequired (?:eng review )?gate (?:for|before) shipping\b/i.test(context) ||
|
||||
/\b(?:unresolved|pending|remaining|outstanding|if|unless|until)\b|\b(?:gap|issue|finding|decision)s?\s+(?:still\s+)?remains?\b/i.test(context) ||
|
||||
/\b(?:not all|not (?:done|complete|resolved)|only after)\b/i.test(context) || conditionalClosure.test(context) ||
|
||||
/(?:^|[.!?;]\s+|\b(?:proceed to|continue to|should|must|will|can|could|would|may|need to)\s+)(?:(?:please|first|then|also)\s+)*(?:add|fix|package|implement|resolve|decide)\b/im.test(context)) return false;
|
||||
}
|
||||
} else {
|
||||
// Some native final menus omit a qid. Require an explicit finished-review
|
||||
// declaration plus resolved findings and the navigation-only question;
|
||||
@@ -1784,7 +1833,10 @@ function isCompletedDxHandoff(call: NativePlanQuestionCall): boolean {
|
||||
/\b(?:unresolved|pending|remaining|outstanding)\b|\b(?:gap|issue|finding|decision)s?\s+(?:still\s+)?remains?\b/i.test(context) ||
|
||||
/(?:^|[.!?;]\s+|\b(?:proceed to|continue to|should|must|will|need to)\s+)(?:(?:please|first|then|also)\s+)*(?:add|fix|package|implement|resolve|decide)\b/im.test(context)) return false;
|
||||
}
|
||||
const labels = q.options.map(o => o.label.trim().replace(/\s*\(recommended\)\s*$/i, ''));
|
||||
const labels = q.options.map(o => {
|
||||
const label = o.label.trim().replace(/\s*\(recommended\)\s*$/i, '');
|
||||
return closedReviewNavigation ? label.replace(/^[A-Z][.)]\s*/i, '') : label;
|
||||
});
|
||||
const runEng = (label: string) => /^Run \/plan-eng-review(?: next)?$/i.test(label);
|
||||
const ready = (label: string) => /^Ready to implement(?:\s*[—–-]\s*run \/devex-review after shipping)?$/i.test(label) ||
|
||||
(resultRecap && /^Start implementing now$/i.test(label));
|
||||
@@ -2663,6 +2715,7 @@ export interface PlanSkillCountObservation {
|
||||
| 'ceiling_reached'
|
||||
| 'silent_write'
|
||||
| 'transcript_unavailable'
|
||||
| 'no_review_questions'
|
||||
| 'exited'
|
||||
| 'timeout';
|
||||
summary: string;
|
||||
@@ -2947,6 +3000,11 @@ export async function runPlanSkillCounting(opts: {
|
||||
// its report/native evidence before input routing; verified gates still
|
||||
// pass the existing silent-write check below. A positively matched
|
||||
// native question above takes precedence over gate text.
|
||||
if (opts.expectedPlanPath && terminalHint === 'plan_ready' && countedCalls.size === 0 &&
|
||||
isQuestionlessNativePlanExit(transcript, opts.expectedPlanPath, startedAt, visible)) {
|
||||
return snapshot('no_review_questions',
|
||||
'Native plan approval reached with zero recorded AskUserQuestion calls; review coverage is missing', visible);
|
||||
}
|
||||
if (opts.expectedPlanPath && terminalHint === 'plan_ready' && !verifiedTerminal) continue;
|
||||
const permission = nativeQuestionVisible || terminalHint === 'plan_ready'
|
||||
? null : filePermission(visible, session.visibleText());
|
||||
|
||||
@@ -21,6 +21,13 @@ real per-example scores plus an overall score. It needs no hosted playground
|
||||
or new interactive UI. Like every first evaluation, it currently waits for the
|
||||
mandatory CI check described in docs/current-contracts.md.
|
||||
|
||||
Expected completed demo output for the bundled sample responses is documented
|
||||
here; the shipped demo prints this per-example and aggregate score format:
|
||||
|
||||
example 1: score=0.80
|
||||
example 2: score=1.00
|
||||
overall: score=0.90
|
||||
|
||||
See docs/api.md for public API and upgrade behavior, and docs/benchmarks.md for
|
||||
the completed onboarding study. These documents describe the existing SDK's
|
||||
behavior; its runtime is maintained separately from this release-planning repo.
|
||||
@@ -42,6 +49,11 @@ On a developer's first local evaluation, the SDK requires a successful remote
|
||||
CI check and blocks for five minutes before returning an evaluation result.
|
||||
There is no skip flag or offline first-run path. The beta plan retains this gate.
|
||||
|
||||
During the required wait, the existing SDK writes a progress line to stderr
|
||||
every 30 seconds, such as "Waiting for CI check: 90s elapsed of 300s", and reports
|
||||
when the check finishes. Progress does not bypass the check or return evaluation
|
||||
results before its required successful completion.
|
||||
|
||||
Authentication errors behave exactly as documented in docs/api.md. All other
|
||||
errors already identify the cause, relevant argument or file, and an actionable
|
||||
fix. Errors redact secrets. API timeouts, cancellation, rate limits, and retries
|
||||
@@ -170,12 +182,56 @@ function substantiveIssue({ header, question, options }: QuestionRecord): boolea
|
||||
const breakingRename = /Client\.evaluate|Client\.run|\bmethod rename\b/i.test(normalized)
|
||||
&& /\b(?:breaking|remov\w*|renam\w*)\b/i.test(normalized)
|
||||
&& /\b(?:migration|deprecation|compatibility|alias|codemod)\b/i.test(normalized);
|
||||
return ciGate || argumentsReversed || opaqueAuth || missingExample || breakingRename;
|
||||
// Expected-output documentation is separate from whether its command
|
||||
// exists. Count the actual gap plus offered documentation remedy, not a
|
||||
// generic navigation question that merely names output in its options.
|
||||
const outputSubject = String.raw`(?:(?:expected|sample|example)(?: demo)?|demo) output`;
|
||||
// Consume the complete noun phrase, including a negating determiner,
|
||||
// before judging its absence. A nested "demo output" suffix cannot
|
||||
// escape "no sample demo output is missing" and become a finding.
|
||||
const missingState = [...normalized.matchAll(new RegExp(String.raw`\b(?:(no|not any)\s+)?${outputSubject}\s+(?:(?:is|are|was|were)\s+)?(?:missing|absent|omitted|unspecified)\b`, 'gi'))];
|
||||
const missingSubject = [...normalized.matchAll(new RegExp(String.raw`\b(?:(no|not any)\s+)?missing\s+${outputSubject}\b`, 'gi'))];
|
||||
const noOutput = new RegExp(String.raw`\bno\s+${outputSubject}\s*(?:[,.;!?]|\b(?:in|from|for|yet)\b)`, 'i');
|
||||
const outputGap = missingState.some(match => !match[1]) || missingSubject.some(match => !match[1]) || noOutput.test(normalized);
|
||||
const missingOutput = /\b(?:README|quick[- ]?start|documentation)\b/i.test(normalized)
|
||||
&& (outputGap || /\b(?:README|quick[- ]?start|documentation)\b[^.!?;]{0,50}\b(?:doesn['’]t|does not)\s+(?:show|include)\b[^.!?;]{0,25}\boutput\b/i.test(normalized))
|
||||
&& Boolean(options?.some(option => /^(?:[A-Z][.:)]\s*)?Add\s+(?:to\s+(?:the\s+)?plan:\s*include\s+)?(?:an?\s+)?(?:expected|sample|example)(?:\s+demo)?\s+output\b[^.!?]*\b(?:README|quick[- ]?start|documentation)\b/i.test(option.label)));
|
||||
return ciGate || argumentsReversed || opaqueAuth || missingExample || breakingRename || missingOutput;
|
||||
}
|
||||
|
||||
/** A setup heading cannot hide a positively selected repair to the existing behavior. */
|
||||
function answeredSetupRepair(fp: AskUserQuestionFingerprint): boolean {
|
||||
const call = fp.nativeCall;
|
||||
if (!call || call.failed || !call.answered || call.questions.length !== 1 ||
|
||||
call.unansweredQuestionIndices?.length || fp.signature !== `${call.sessionId}:${call.toolUseId}`) return false;
|
||||
const question = call.questions[0]!;
|
||||
if (question.multiSelect) return false;
|
||||
const selected = question.options.filter(option => option.label === call.answers?.[question.question]);
|
||||
if (selected.length !== 1) return false;
|
||||
const ids = [...question.question.matchAll(/<gstack-qid:([a-z0-9-]+)>/gi)];
|
||||
if (ids.length !== 1 || (question.question.match(/<gstack-qid/gi)?.length ?? 0) !== 1) return false;
|
||||
const id = ids[0]![1]!.toLowerCase();
|
||||
const header = question.header.trim().replace(/^D\s*\d+\s*[—–:-]\s*/i, '');
|
||||
const text = question.question.replace(/\s+/g, ' ');
|
||||
const label = selected[0]!.label.replace(/^[A-Z][.):]\s*/i, '');
|
||||
if (id === 'plan-devex-review-tthw-tier' && /^TTHW target$/i.test(header)) {
|
||||
return /TTHW|Time-to-Hello-World/i.test(text) && /\bCI\b/i.test(text) &&
|
||||
/\b(?:mandatory|blocks?|retains? the CI block)\b/i.test(text) &&
|
||||
/(?:^|[—–:]\s*)add\s+(?:an?\s+)?(?:skip flag|--skip-ci|offline(?:[- ]first[- ]run)? path)\b/i.test(label);
|
||||
}
|
||||
if (id === 'plan-devex-review-magical-moment' && /^Magical moment$/i.test(header)) {
|
||||
// The selected option adds progress feedback beyond the already chosen
|
||||
// demo vehicle and prior CI-bypass decision. An unselected remedy or
|
||||
// a confirmation of that vehicle alone remains setup.
|
||||
return /\bdemo\b/i.test(text) && /\bsilently blocks?\b|\bsilent (?:CI )?wait\b/i.test(text) &&
|
||||
/(?:^|[—–:]\s*)add\s+[^.!?;]{0,80}\bprogress (?:output|indicator)\b/i.test(label);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** A batched native call remains one decision; the caller owns call-ID deduplication. */
|
||||
export function isDevexReviewIssue(fp: AskUserQuestionFingerprint): boolean {
|
||||
return questionRecords(fp, true).some(substantiveIssue);
|
||||
return answeredSetupRepair(fp) || questionRecords(fp, true).some(substantiveIssue);
|
||||
}
|
||||
|
||||
/** Select POLISH only on the recognized mode menu; leave all other answers unchanged. */
|
||||
|
||||
@@ -254,7 +254,9 @@ let cachedSkillsConfigDir: string | null = null;
|
||||
* user scope, mirroring ./setup's registration exactly: each discovered skill
|
||||
* gets a REAL directory `<configDir>/skills/<registryName>/` containing a
|
||||
* SYMLINK to that skill's SKILL.md (absolute path), plus symlinks to its
|
||||
* runtime assets using setup's exclusions. registryName is the frontmatter `name:`
|
||||
* runtime assets using setup's exclusions. The gstack checkout itself also
|
||||
* lives at `<configDir>/skills/gstack`, as in an installed setup, so canonical
|
||||
* lazy-section paths work alongside flattened discovery. registryName is the frontmatter `name:`
|
||||
* (dir-name fallback), NO gstack- prefix; the root SKILL.md router registers
|
||||
* as `_gstack-command`. skillCensus().registryEntries is the authoritative
|
||||
* set of what must appear here.
|
||||
@@ -316,6 +318,10 @@ export function hermeticSkillsConfigDir(): string {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Claude can resolve canonical skill paths from CLAUDE_CONFIG_DIR rather
|
||||
// than HOME. Expose the same runtime checkout here as well as the flattened
|
||||
// entries above; never send a missing lazy-section lookup to operator state.
|
||||
fs.symlinkSync(root, path.join(skillsDir, 'gstack'), 'dir');
|
||||
cachedSkillsConfigDir = configDir;
|
||||
return configDir;
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ export function hermeticSkillRuntime(): { home: string; root: string } {
|
||||
// tools may resolve ~/paths even though slash commands use the latter.
|
||||
const registry = path.join(hermeticSkillsConfigDir(), 'skills');
|
||||
for (const name of fs.readdirSync(registry)) {
|
||||
if (name === 'gstack') continue; // canonical checkout already linked above
|
||||
fs.symlinkSync(path.join(registry, name), path.join(path.dirname(root), name), 'dir');
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
@@ -139,7 +139,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// Each one tests behavior the SDK harness can't observe (rendered TTY,
|
||||
// numbered-option lists, multi-phase ordering, idempotency state echo).
|
||||
'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-ask-user-question-format-compliance.test.ts'],
|
||||
'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/ceo-mode-option.ts', 'test/ceo-mode-option.test.ts', 'test/pty-option-selection.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-ceo-mode-routing.test.ts', 'test/ceo-mode-posture-native.test.ts', 'test/fixtures/ceo-hold-posture-l.json', 'test/ceo-mode-labels-native.test.ts', 'test/fixtures/ceo-mode-labels-l.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt'],
|
||||
'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/ceo-mode-option.ts', 'test/ceo-mode-option.test.ts', 'test/pty-option-selection.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-ceo-mode-routing.test.ts', 'test/ceo-mode-posture-native.test.ts', 'test/fixtures/ceo-hold-posture-l.json', 'test/ceo-mode-labels-native.test.ts', 'test/fixtures/ceo-mode-labels-l.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/ceo-mode-prerequisite.test.ts', 'test/fixtures/ceo-mode-prerequisite-o-calls.json'],
|
||||
'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'],
|
||||
'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-ship-idempotency.test.ts'],
|
||||
'tpa-present': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
|
||||
@@ -154,16 +154,16 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// the selector sets GSTACK_CARVE_SKILL=<name> to scope cost to the changed
|
||||
// skill (D-CODEX A). Touching the registry/helper or sections.ts runs all.
|
||||
'carve-section-loading': ['scripts/resolvers/composition.ts', 'test/autoplan-review-discovery.test.ts', 'test/autoplan-phase-order.test.ts', 'design-html/**', 'design-shotgun/**', 'qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/resolvers/redact-doc.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/session-runner-tools.test.ts', 'test/carve-section-loading.test.ts', 'bin/gstack-autoplan-snapshot.ts', 'test/autoplan-snapshot.test.ts'],
|
||||
'autoplan-chain-pty': ['scripts/resolvers/composition.ts', 'test/autoplan-review-discovery.test.ts', 'test/autoplan-phase-order.test.ts', 'autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/autoplan-dashboard.md', 'test/autoplan-chain-fixture.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'bin/gstack-config', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/autoplan-setup-question.ts', 'test/autoplan-setup-question.test.ts', 'test/helpers/autoplan-phase-observer.ts', 'test/autoplan-phase-observer.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/skill-e2e-autoplan-chain.test.ts', 'bin/gstack-autoplan-snapshot.ts', 'test/autoplan-snapshot.test.ts', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/helpers/eval-budgets.ts', 'test/autoplan-eval-budget.test.ts', 'test/eval-budgets-policy.test.ts', 'test/eval-detach-timeout-floor.test.ts', 'scripts/test-paid-shards.ts', '.github/workflows/evals-periodic.yml', 'test/fixtures/autoplan-routing-n-screen.txt'],
|
||||
'autoplan-chain-pty': ['scripts/resolvers/composition.ts', 'test/autoplan-review-discovery.test.ts', 'test/autoplan-phase-order.test.ts', 'autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/autoplan-dashboard.md', 'test/autoplan-chain-fixture.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'bin/gstack-config', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/autoplan-setup-question.ts', 'test/autoplan-setup-question.test.ts', 'test/helpers/autoplan-phase-observer.ts', 'test/autoplan-phase-observer.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/skill-e2e-autoplan-chain.test.ts', 'bin/gstack-autoplan-snapshot.ts', 'test/autoplan-snapshot.test.ts', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/helpers/eval-budgets.ts', 'test/autoplan-eval-budget.test.ts', 'test/eval-budgets-policy.test.ts', 'test/eval-detach-timeout-floor.test.ts', 'scripts/test-paid-shards.ts', '.github/workflows/evals-periodic.yml', 'test/fixtures/autoplan-routing-n-screen.txt', 'test/autoplan-routing-o.test.ts', 'test/fixtures/autoplan-routing-o-screen.txt', 'test/autoplan-setup-packet-o.test.ts', 'test/fixtures/autoplan-setup-packet-o-screen.txt', 'test/fixtures/autoplan-setup-packet-o-call.json'],
|
||||
|
||||
// Per-finding AskUserQuestion count + review-report-at-bottom assertion.
|
||||
// Each test drives its skill end-to-end; touchfiles include preamble +
|
||||
// completion-status resolvers because they affect question cadence and
|
||||
// terminal output (the regression surface this test catches).
|
||||
'plan-ceo-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'test/skill-ceo-section-ordering.test.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/ceo-completion-handoff.ts', 'test/ceo-completion-handoff.test.ts', 'test/fixtures/ceo-completion-handoff-calls.json', 'test/fixtures/ceo-completion-handoff-j-calls.json', 'test/fixtures/ceo-completion-handoff-k-calls.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-ceo-finding-count.test.ts', 'test/ceo-completion-handoff-l.test.ts', 'test/fixtures/ceo-completion-handoff-l-calls.json', 'test/ceo-completion-handoff-m.test.ts', 'test/fixtures/ceo-completion-handoff-m-call.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/ceo-handoff-n-calls.json'],
|
||||
'plan-eng-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-eng-finding-count.test.ts', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt'],
|
||||
'plan-design-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/design-count-review.ts', 'test/design-count-review.test.ts', 'test/fixtures/design-review-j-calls.json', 'test/helpers/design-count-outside.ts', 'test/design-count-outside.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts', 'test/fixtures/design-review-l-calls.json', 'test/design-completion-handoff.test.ts', 'test/fixtures/design-handoff-l-calls.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/design-review-n-calls.json', 'test/design-completion-handoff-scored.test.ts', 'test/fixtures/design-handoff-n-calls.json'],
|
||||
'plan-devex-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/devex-count-fixture.ts', 'test/devex-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/devex-review-n-calls.json'],
|
||||
'plan-ceo-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'test/skill-ceo-section-ordering.test.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/ceo-completion-handoff.ts', 'test/ceo-completion-handoff.test.ts', 'test/fixtures/ceo-completion-handoff-calls.json', 'test/fixtures/ceo-completion-handoff-j-calls.json', 'test/fixtures/ceo-completion-handoff-k-calls.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-ceo-finding-count.test.ts', 'test/ceo-completion-handoff-l.test.ts', 'test/fixtures/ceo-completion-handoff-l-calls.json', 'test/ceo-completion-handoff-m.test.ts', 'test/fixtures/ceo-completion-handoff-m-call.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/ceo-handoff-n-calls.json', 'test/ceo-completion-handoff-o.test.ts', 'test/fixtures/ceo-completion-handoff-o-call.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json'],
|
||||
'plan-eng-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/skill-e2e-plan-eng-finding-count.test.ts', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/plan-count-empty-review.test.ts', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json'],
|
||||
'plan-design-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/design-count-review.ts', 'test/design-count-review.test.ts', 'test/fixtures/design-review-j-calls.json', 'test/helpers/design-count-outside.ts', 'test/design-count-outside.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts', 'test/fixtures/design-review-l-calls.json', 'test/design-completion-handoff.test.ts', 'test/fixtures/design-handoff-l-calls.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/design-review-n-calls.json', 'test/design-completion-handoff-scored.test.ts', 'test/fixtures/design-handoff-n-calls.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json'],
|
||||
'plan-devex-finding-count': ['bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/devex-count-fixture.ts', 'test/devex-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/devex-review-n-calls.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/plan-count-design-questionless-report.md', 'test/devex-output-o.test.ts', 'test/fixtures/devex-review-o-calls.json', 'test/fixtures/devex-output-o-retry-call.json', 'test/devex-setup-remedy-o.test.ts', 'test/fixtures/devex-review-o-retry-calls.json', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json'],
|
||||
|
||||
// Gate-tier reviewCount-floor counterparts. Catch the May 2026 transcript
|
||||
// bug (model wrote a plan-mode plan and ExitPlanMode'd without firing any
|
||||
|
||||
@@ -60,11 +60,13 @@ const record = {
|
||||
codexHome, codexConfig:fs.readFileSync(path.join(codexHome,'config.toml'),'utf8'),
|
||||
codexAuthPreserved:JSON.parse(fs.readFileSync(path.join(codexHome,'auth.json'),'utf8')).fixture_auth === true,
|
||||
browserCache:process.env.PLAYWRIGHT_BROWSERS_PATH,
|
||||
discovery: ['plan-devex-review/dx-hall-of-fame.md', 'plan-devex-review/sections/review-sections.md', 'review/checklist.md'].map(relative => {
|
||||
discovery: ['plan-design-review/sections/review-sections.md', 'plan-devex-review/dx-hall-of-fame.md', 'plan-devex-review/sections/review-sections.md', 'review/checklist.md'].map(relative => {
|
||||
const homePath = path.join(process.env.HOME, '.claude', 'skills', relative);
|
||||
const configPath = path.join(process.env.CLAUDE_CONFIG_DIR || path.join(process.env.HOME, '.claude'), 'skills', relative);
|
||||
const configRuntimePath = path.join(process.env.CLAUDE_CONFIG_DIR || path.join(process.env.HOME, '.claude'), 'skills', 'gstack', relative);
|
||||
return {relative, home:fs.existsSync(homePath) ? fs.realpathSync(homePath) : null,
|
||||
config:fs.existsSync(configPath) ? fs.realpathSync(configPath) : null};
|
||||
config:fs.existsSync(configPath) ? fs.realpathSync(configPath) : null,
|
||||
configRuntime:fs.existsSync(configRuntimePath) ? fs.realpathSync(configRuntimePath) : null};
|
||||
}),
|
||||
};
|
||||
fs.writeFileSync(process.env.RUNTIME_RECORD,JSON.stringify(record));
|
||||
@@ -129,6 +131,7 @@ try {
|
||||
for (const asset of result.discovery) {
|
||||
expect(asset.home, item.name + ': HOME discovery ' + asset.relative).toBe(fs.realpathSync(path.join(ROOT,asset.relative)));
|
||||
expect(asset.config, item.name + ': config discovery ' + asset.relative).toBe(asset.home);
|
||||
expect(asset.configRuntime, item.name + ': config runtime discovery ' + asset.relative).toBe(asset.home);
|
||||
}
|
||||
const expectedCodex = item.overrides?.CODEX_HOME !== undefined
|
||||
? item.overrides.CODEX_HOME || codexDefault : item.configuredCodex || codexDefault;
|
||||
|
||||
@@ -9,7 +9,8 @@
|
||||
* /.claude suffix, under the hermetic runRoot).
|
||||
* 2. Registration mirrors ./setup exactly: one entry per
|
||||
* skillCensus().registryEntries, each a REAL dir with a SKILL.md symlink
|
||||
* resolving to a real file (plus sections/ when the skill has one).
|
||||
* resolving to a real file (plus sections/ when the skill has one), and the
|
||||
* same canonical gstack runtime checkout used by lazy-section paths.
|
||||
* 3. connect-chrome (dir symlink) collapses into open-gstack-browser — no
|
||||
* duplicate, no connect-chrome entry.
|
||||
* 4. Per-process idempotence: the second call returns the cached dir.
|
||||
@@ -36,13 +37,18 @@ describe('hermeticSkillsConfigDir', () => {
|
||||
expect(configDir.startsWith(getHermeticDirs().runRoot + path.sep)).toBe(true);
|
||||
});
|
||||
|
||||
test('one registry entry per skillCensus registryEntries, nothing extra', () => {
|
||||
test('exact skill registry plus the canonical runtime checkout', () => {
|
||||
const seeded = fs.readdirSync(skillsDir).sort();
|
||||
expect(seeded).toEqual(skillCensus(ROOT).registryEntries);
|
||||
expect(seeded).toEqual([...skillCensus(ROOT).registryEntries, 'gstack'].sort());
|
||||
const runtime = path.join(skillsDir, 'gstack');
|
||||
expect(fs.lstatSync(runtime).isSymbolicLink()).toBe(true);
|
||||
expect(fs.realpathSync(runtime)).toBe(fs.realpathSync(ROOT));
|
||||
expect(fs.readFileSync(path.join(runtime, 'plan-design-review/sections/review-sections.md'), 'utf8'))
|
||||
.toBe(fs.readFileSync(path.join(ROOT, 'plan-design-review/sections/review-sections.md'), 'utf8'));
|
||||
});
|
||||
|
||||
test('every SKILL.md is a symlink resolving to a real file', () => {
|
||||
for (const entry of fs.readdirSync(skillsDir)) {
|
||||
for (const entry of skillCensus(ROOT).registryEntries) {
|
||||
const link = path.join(skillsDir, entry, 'SKILL.md');
|
||||
expect(fs.lstatSync(link).isSymbolicLink()).toBe(true);
|
||||
expect(fs.statSync(link).isFile()).toBe(true); // follows the link
|
||||
|
||||
@@ -155,6 +155,11 @@ describe('hermetic wiring tripwire', () => {
|
||||
const skillsDir = path.join(configDir, 'skills');
|
||||
const repoRootReal = fs.realpathSync(ROOT) + path.sep;
|
||||
for (const entry of fs.readdirSync(skillsDir)) {
|
||||
if (entry === 'gstack') {
|
||||
expect(fs.lstatSync(path.join(skillsDir, entry)).isSymbolicLink()).toBe(true);
|
||||
expect(fs.realpathSync(path.join(skillsDir, entry)) + path.sep).toBe(repoRootReal);
|
||||
continue;
|
||||
}
|
||||
const target = fs.readlinkSync(path.join(skillsDir, entry, 'SKILL.md'));
|
||||
const resolved = fs.realpathSync(target);
|
||||
// Targets inside the live repo checkout are the blessed edge — exempt
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { hasNativePlanTerminal } from './helpers/claude-pty-runner';
|
||||
import type { NativePlanQuestionCall, PlanCountTranscript } from './helpers/plan-count-transcript';
|
||||
import fixture from './fixtures/devex-handoff-o-call.json';
|
||||
|
||||
function replay(mutate?: (call: NativePlanQuestionCall, transcript: PlanCountTranscript) => void): boolean {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'dx-native-handoff-'));
|
||||
const report = path.join(dir, 'review.md');
|
||||
const transcript = structuredClone(fixture.transcript) as PlanCountTranscript;
|
||||
const call = transcript.calls.at(-1)!;
|
||||
mutate?.(call, transcript);
|
||||
fs.writeFileSync(report, fixture.report);
|
||||
const writtenAt = Date.parse(fixture.reportWrite.resultAt);
|
||||
fs.utimesSync(report, writtenAt / 1000, writtenAt / 1000);
|
||||
try {
|
||||
return hasNativePlanTerminal(transcript, report,
|
||||
Date.parse(fixture.startedAt), 'plan_ready');
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test('actual completed DX navigation preserves the prior full report and all thirteen native calls', () => {
|
||||
expect(fixture.transcript.calls).toHaveLength(13);
|
||||
expect(fixture.transcript.calls.every(call => call.answered && !call.failed)).toBe(true);
|
||||
expect(fixture.reportWrite.failed).toBe(false);
|
||||
expect(replay()).toBe(true);
|
||||
});
|
||||
|
||||
test('closed navigation cannot hide a real new issue, unfinished review, or conditional closure', () => {
|
||||
const variants = [
|
||||
(q: any) => { q.header = 'New finding'; },
|
||||
(q: any) => { q.question = q.question.replace('D11 — What next?', 'D11 — Should we fix the missing authorization check?'); },
|
||||
(q: any) => { q.question = q.question.replace('The DX review is done.', 'The DX review is not done.'); },
|
||||
(q: any) => { q.question = q.question.replace('The DX review is done.', 'The DX review is done only after tests pass.'); },
|
||||
(q: any) => { q.question += '\nOne documentation gap remains unresolved.'; },
|
||||
(q: any) => { q.question += '\nPlease fix the missing authorization check first.'; },
|
||||
(q: any) => { q.question += '\nShould we add the missing test?'; },
|
||||
(q: any) => { q.question += '\nOnce the tests pass, the DX review will be complete.'; },
|
||||
(q: any) => { q.question += '\nAll decisions resolved after tests pass.'; },
|
||||
(q: any) => { q.question += '\nDX review complete after tests pass.'; },
|
||||
(q: any) => { q.question += '\nWe could fix the missing authorization check before Eng.'; },
|
||||
(q: any) => { q.question = q.question.replace('ELI10: The DX review is done.', '```text\nThe DX review is done.\n```'); },
|
||||
(q: any) => { q.question = q.question.replace('ELI10: The DX review is done.', '```text\nThe DX review is done.'); },
|
||||
(q: any) => { q.question = q.question.replace('ELI10: The DX review is done.', '> The DX review is done.'); },
|
||||
(q: any) => { q.options[2].description = 'Proceed to fix the missing contrast test before Eng.'; },
|
||||
(q: any) => { q.options[2].description = 'One issue remains unresolved; handle it manually.'; },
|
||||
(q: any) => { q.options[2].label = 'C) Add the missing test'; },
|
||||
(q: any) => { q.options.push({ label: 'D) Add a migration guide', description: 'A new required deliverable.' }); },
|
||||
(q: any) => { q.question = q.question.replace('plan-devex-review-next-steps', 'plan-devex-review-new-issue'); },
|
||||
(q: any) => { q.question += '\n<gstack-qid:broken'; },
|
||||
(q: any) => { q.multiSelect = true; },
|
||||
];
|
||||
for (const mutate of variants) {
|
||||
expect(replay(call => {
|
||||
const question = call.questions[0]!;
|
||||
mutate(question);
|
||||
// Keep a real offered answer after text mutations so the semantic guard,
|
||||
// rather than a stale answer key, is what must reject the altered call.
|
||||
call.answers = { [question.question]: question.options[0]!.label };
|
||||
}), String(mutate)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('native failure, offered answer, report freshness and real Exit remain required', () => {
|
||||
expect(replay(call => { call.answered = false; })).toBe(false);
|
||||
expect(replay(call => { call.failed = true; })).toBe(false);
|
||||
expect(replay(call => { call.unansweredQuestionIndices = [0]; })).toBe(false);
|
||||
expect(replay(call => { call.answers = { [call.questions[0]!.question]: 'Add a new test first' }; })).toBe(false);
|
||||
expect(replay((_call, transcript) => { transcript.planReadyRequests = []; })).toBe(false);
|
||||
expect(replay((_call, transcript) => { transcript.planReadyRequests![0]!.failed = true; })).toBe(false);
|
||||
expect(replay((_call, transcript) => { transcript.planReadyRequests![0]!.sessionId = 'foreign'; })).toBe(false);
|
||||
expect(replay((_call, transcript) => {
|
||||
transcript.calls[11]!.answeredAt = new Date(Date.parse(fixture.reportWrite.resultAt) + 1000).toISOString();
|
||||
})).toBe(false);
|
||||
});
|
||||
|
||||
test('pure navigation preserves actual option order and ordinary next-review sequencing', () => {
|
||||
expect(replay(call => { call.questions[0]!.options.reverse(); })).toBe(true);
|
||||
expect(replay(call => {
|
||||
const q = call.questions[0]!;
|
||||
q.options[0]!.description += ' After Eng review is complete, proceed to implementation.';
|
||||
})).toBe(true);
|
||||
});
|
||||
@@ -0,0 +1,146 @@
|
||||
/** A native approval gate with no questions is missing coverage, not a timeout or pass. */
|
||||
import { expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { isQuestionlessNativePlanExit, hasNativePlanTerminal } from './helpers/claude-pty-runner';
|
||||
import type { PlanCountTranscript } from './helpers/plan-count-transcript';
|
||||
|
||||
// Exact caller-owned report from a native zero-question run; the Run header is
|
||||
// intentional. Diagnostic failure must not depend on the positive coverage header.
|
||||
const GATE = 'Exit plan mode?\n\nClaude wants to exit plan mode\n❯ 1. Yes, and switch to default (ask each time) for this session\n 2. No\n';
|
||||
const REPORT = fs.readFileSync(path.join(import.meta.dir, 'fixtures/plan-count-design-questionless-report.md'), 'utf8');
|
||||
|
||||
test.skipIf(process.platform === 'win32')('real PTY fails promptly at an owned native exit with no recorded questions', async () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'empty-native-review-'));
|
||||
const fake = path.join(dir, 'fake-claude');
|
||||
const worker = path.join(dir, 'worker.ts');
|
||||
const report = path.join(dir, 'plan.md');
|
||||
const inputs = path.join(dir, 'inputs.jsonl');
|
||||
const pidFile = path.join(dir, 'pid');
|
||||
const resultFile = path.join(dir, 'result.json');
|
||||
const runner = pathToFileURL(path.join(import.meta.dir, 'helpers/claude-pty-runner.ts')).href;
|
||||
fs.writeFileSync(fake, `#!${process.execPath}\n` + String.raw`
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
const stat = process.platform === 'linux' ? fs.readFileSync('/proc/self/stat', 'utf8') : null;
|
||||
fs.writeFileSync(process.env.PROBE_PID, JSON.stringify({pid:process.pid,
|
||||
startTicks:stat ? stat.slice(stat.lastIndexOf(')') + 2).split(' ')[19] : null}));
|
||||
const sessionId = 'empty-native-review';
|
||||
const project = path.join(process.env.CLAUDE_CONFIG_DIR, 'projects', sessionId);
|
||||
fs.mkdirSync(project, { recursive: true });
|
||||
const transcript = path.join(project, sessionId + '.jsonl');
|
||||
let sent = false;
|
||||
process.stdin.setRawMode?.(true);
|
||||
process.stdin.on('data', data => {
|
||||
fs.appendFileSync(process.env.PROBE_INPUTS, JSON.stringify(data.toString()) + '\n');
|
||||
if (sent) return; sent = true;
|
||||
fs.writeFileSync(process.env.PROBE_PLAN, process.env.PROBE_REPORT);
|
||||
setTimeout(() => {
|
||||
fs.writeFileSync(transcript, JSON.stringify({cwd:process.cwd(), sessionId, isSidechain:false,
|
||||
timestamp:new Date().toISOString(), message:{role:'assistant', content:[
|
||||
{type:'tool_use', id:'empty-review-exit', name:'ExitPlanMode', input:{}}
|
||||
]}}) + '\n');
|
||||
process.stdout.write('Exit plan mode?\n\nClaude wants to exit plan mode\n❯ 1. Yes, and switch to default (ask each time) for this session\n 2. No\n');
|
||||
}, 30);
|
||||
});
|
||||
process.stdin.resume();
|
||||
`);
|
||||
fs.chmodSync(fake, 0o755);
|
||||
fs.writeFileSync(worker, `import { runPlanSkillCounting } from ${JSON.stringify(runner)};\n` +
|
||||
`if (process.env.BROWSE_TERMINAL_BINARY !== ${JSON.stringify(fake)}) throw new Error('fake CLI not selected');\n` +
|
||||
`const result = await runPlanSkillCounting({skillName:'plan-design-review',slashCommand:'/plan-design-review',followUpPrompt:'# Empty review fixture',expectedPlanPath:${JSON.stringify(report)},isLastStep0AUQ:()=>false,reviewCountCeiling:8,timeoutMs:20000,env:${JSON.stringify({PROBE_PLAN:report,PROBE_INPUTS:inputs,PROBE_PID:pidFile,PROBE_REPORT:REPORT})}});\n` +
|
||||
`await Bun.write(${JSON.stringify(resultFile)},JSON.stringify(result));\n`);
|
||||
const child = Bun.spawn([process.execPath, worker], {
|
||||
env: { ...process.env, EVALS_HERMETIC:'1', EVALS_RUN_ID:'', BROWSE_TERMINAL_BINARY:fake },
|
||||
stdout:'pipe', stderr:'pipe',
|
||||
});
|
||||
const timer = setTimeout(() => child.kill('SIGKILL'), 35000);
|
||||
try {
|
||||
const [code, out, err] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]);
|
||||
expect(code, out + err).toBe(0);
|
||||
const result = JSON.parse(fs.readFileSync(resultFile, 'utf8'));
|
||||
expect(result.outcome).toBe('no_review_questions');
|
||||
expect(result.reviewCount).toBe(0);
|
||||
expect(result.step0Count).toBe(0);
|
||||
expect(result.transcript.calls).toHaveLength(0);
|
||||
expect(result.transcript.planReadyRequests).toHaveLength(1);
|
||||
expect(fs.readFileSync(inputs,'utf8').trim().split('\n').map(line => JSON.parse(line))).toEqual(['/plan-design-review\r']);
|
||||
expect(() => process.kill(JSON.parse(fs.readFileSync(pidFile,'utf8')).pid, 0)).toThrow();
|
||||
} finally {
|
||||
clearTimeout(timer); child.kill('SIGKILL');
|
||||
// An outer watchdog can outlive the worker's session cleanup. Only the
|
||||
// uniquely named fake executable may be reaped; a reused PID is not owned.
|
||||
if (fs.existsSync(pidFile)) try {
|
||||
const saved = JSON.parse(fs.readFileSync(pidFile,'utf8'));
|
||||
if (Number.isInteger(saved.pid) && saved.pid > 0) {
|
||||
let owned = false;
|
||||
if (process.platform === 'linux') {
|
||||
const stat = fs.readFileSync(`/proc/${saved.pid}/stat`, 'utf8');
|
||||
owned = stat.slice(stat.lastIndexOf(')') + 2).split(' ')[19] === saved.startTicks &&
|
||||
fs.readFileSync(`/proc/${saved.pid}/cmdline`, 'utf8').split('\0').includes(fake);
|
||||
} else {
|
||||
owned = execFileSync('ps', ['-p', String(saved.pid), '-o', 'command='], {encoding:'utf8',timeout:5000}).includes(fake);
|
||||
}
|
||||
if (owned) process.kill(saved.pid, 'SIGKILL');
|
||||
}
|
||||
} catch { /* the owned fake child already exited */ }
|
||||
fs.rmSync(dir, { recursive:true, force:true });
|
||||
}
|
||||
}, 40000);
|
||||
|
||||
|
||||
test('questionless exit is only a failure diagnostic with fresh report and coherent native identity', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'questionless-report-'));
|
||||
const report = path.join(dir, 'plan.md');
|
||||
const start = Date.now() - 10000;
|
||||
const written = start + 1000;
|
||||
const readyAt = start + 2000;
|
||||
fs.writeFileSync(report, REPORT);
|
||||
fs.utimesSync(report, written / 1000, written / 1000);
|
||||
const base: PlanCountTranscript = { status:'ready', calls:[],
|
||||
assistantMessages:[{sessionId:'owned',timestamp:new Date(readyAt).toISOString(),text:'Design report written.'}],
|
||||
planReadyRequests:[{sessionId:'owned',toolUseId:'ready',timestamp:new Date(readyAt).toISOString(),failed:false}] };
|
||||
try {
|
||||
expect(isQuestionlessNativePlanExit(base, report, start, GATE)).toBe(true);
|
||||
expect(hasNativePlanTerminal(base, report, start, 'plan_ready')).toBe(false);
|
||||
for (const screen of ['', '> Would you like to proceed?',
|
||||
'● Still validating the report. It will be ready to execute after checks.\n✻ Working…',
|
||||
'Example:\n' + GATE, '```text\n' + GATE + '```',
|
||||
GATE + '\nThinking...', GATE.replace(' 2. No\n', ''),
|
||||
GATE.split('\n').map(line => '> ' + line).join('\n')]) {
|
||||
expect(isQuestionlessNativePlanExit(base, report, start, screen)).toBe(false);
|
||||
}
|
||||
for (const mutate of [
|
||||
(t: any) => { t.status = 'missing'; },
|
||||
(t: any) => { t.status = 'error'; },
|
||||
(t: any) => { t.calls = [{answered:false,failed:false}]; },
|
||||
(t: any) => { t.calls = [{answered:true,failed:false}]; },
|
||||
(t: any) => { t.planReadyRequests = []; },
|
||||
(t: any) => { t.planReadyRequests[0].failed = true; },
|
||||
(t: any) => { t.planReadyRequests[0].sessionId = ''; },
|
||||
(t: any) => { t.planReadyRequests[0].toolUseId = ''; },
|
||||
(t: any) => { t.planReadyRequests[0].sessionId = 'foreign'; },
|
||||
(t: any) => { t.planReadyRequests[0].timestamp = 'invalid'; },
|
||||
(t: any) => { t.planReadyRequests[0].timestamp = new Date(start - 1).toISOString(); },
|
||||
(t: any) => { t.planReadyRequests[0].timestamp = new Date(Date.now() + 10000).toISOString(); },
|
||||
(t: any) => { t.planReadyRequests.push({...t.planReadyRequests[0], timestamp:new Date(readyAt + 1).toISOString(),failed:true}); },
|
||||
]) {
|
||||
const changed = structuredClone(base); mutate(changed);
|
||||
expect(isQuestionlessNativePlanExit(changed, report, start, GATE)).toBe(false);
|
||||
}
|
||||
for (const at of [start - 1, readyAt + 1]) {
|
||||
fs.utimesSync(report, at / 1000, at / 1000);
|
||||
expect(isQuestionlessNativePlanExit(base, report, start, GATE)).toBe(false);
|
||||
}
|
||||
for (const body of ['# Pending report', '```markdown\n' + REPORT + '\n```', REPORT + '\n## More work']) {
|
||||
fs.writeFileSync(report, body); fs.utimesSync(report, written / 1000, written / 1000);
|
||||
expect(isQuestionlessNativePlanExit(base, report, start, GATE)).toBe(false);
|
||||
}
|
||||
expect(isQuestionlessNativePlanExit(base, 'plan.md', start, GATE)).toBe(false);
|
||||
fs.rmSync(report);
|
||||
expect(isQuestionlessNativePlanExit(base, report, start, GATE)).toBe(false);
|
||||
} finally { fs.rmSync(dir, {recursive:true,force:true}); }
|
||||
});
|
||||
@@ -151,7 +151,8 @@ describeE2E('/autoplan native chain ordering (periodic)', () => {
|
||||
// deduplicates the complete question before returning an input.
|
||||
const setup = autoplanSetupDecision(visible, seenSetupQuestions, transcript.calls.find(call => !call.answered && !call.failed));
|
||||
if (setup.kind === 'input') {
|
||||
if (setup.input.includes('\r')) await selectPtyNumberedOption(session, Number(setup.input.trim()));
|
||||
if (setup.input === '\r') session.send(setup.input); // Verified setup-packet Submit, no numbered choice.
|
||||
else if (setup.input.includes('\r')) await selectPtyNumberedOption(session, Number(setup.input.trim()));
|
||||
else session.send(setup.input);
|
||||
for (const signature of setup.signatures) seenSetupQuestions.add(signature);
|
||||
await Bun.sleep(2000);
|
||||
|
||||
@@ -81,7 +81,8 @@ const PLAN = [
|
||||
].join('\n');
|
||||
|
||||
/**
|
||||
* Navigate prior AskUserQuestions by picking option 1 until we hit an AskUserQuestion whose
|
||||
* Navigate prior AskUserQuestions with the existing prerequisite skip, otherwise option 1,
|
||||
* until we hit an AskUserQuestion whose
|
||||
* options match one of the 4 mode names. Returns the option index
|
||||
* matching `targetMode`, with the buffer marker pointing AT that AskUserQuestion.
|
||||
*
|
||||
@@ -129,7 +130,7 @@ async function navigateToModeAskUserQuestion(
|
||||
return { modeIndex: action.index, visibleAtMode: visible, question: action.question };
|
||||
}
|
||||
|
||||
// Not the mode AskUserQuestion — answer with option 1 (recommended) and continue.
|
||||
// Follow the captured prerequisite choice; other navigation still uses option 1.
|
||||
if (priorAnswered >= maxNav) {
|
||||
throw new Error(
|
||||
`Navigated ${maxNav} prior AskUserQuestions without reaching the mode AskUserQuestion. ` +
|
||||
@@ -138,7 +139,7 @@ async function navigateToModeAskUserQuestion(
|
||||
);
|
||||
}
|
||||
priorAnswered++;
|
||||
session.send(planCountQuestionInput(visible, action.question, 1));
|
||||
session.send(planCountQuestionInput(visible, action.question, action.index));
|
||||
// Give the agent a beat to advance before re-polling.
|
||||
await Bun.sleep(2000);
|
||||
}
|
||||
|
||||
@@ -31,7 +31,8 @@ const CEILING = N + 2; // 7
|
||||
// coverage gaps when these surrounding contracts were omitted. The shared
|
||||
// mutable state and missing legacy regression below remain deliberate defects.
|
||||
const planEng5Findings = (planPath: string) => [
|
||||
`Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`,
|
||||
`Please review this plan thoroughly. Write the full reviewed implementation plan, including its final ## GSTACK REVIEW REPORT section, to ${planPath} (use Edit/Write to that exact path).`,
|
||||
`The separate QA Test Plan artifact belongs at the skill-prescribed test-plan path; keep this requested deliverable as the full reviewed implementation plan.`,
|
||||
'',
|
||||
'# Plan: Multi-tenant Auth Refactor',
|
||||
'',
|
||||
|
||||
Reference in new issue
Block a user