mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-07 01:15:47 +01:00
test: recognize completed review findings and scoped artifact permissions
This commit is contained in:
Garry Tan committed
1 parent
f26d569e03
commit
5301119aa8
27 files changed
+5492
-49
No files matched your search
@@ -0,0 +1,125 @@
|
||||
import {expect,test} from 'bun:test';
|
||||
import fs from 'node:fs';import os from 'node:os';import path from 'node:path';
|
||||
import fixture from './fixtures/autoplan-artifact-stall-as.json';
|
||||
import * as permission from './helpers/autoplan-artifact-permission';
|
||||
import {readPendingAutoplanArtifact,autoplanArtifactRecorderStatus} from './helpers/autoplan-artifact-recorder';
|
||||
import {readPlanCountTranscript,type NativePublicToolEvent} from './helpers/plan-count-transcript';
|
||||
import {E2E_TOUCHFILES,selectTests} from './helpers/touchfiles';
|
||||
|
||||
function replay() {
|
||||
const root=fs.mkdtempSync(path.join(os.tmpdir(),'gstack-ap-stall-'));
|
||||
const runtimeBefore=path.dirname(path.dirname(fixture.stateRoot));
|
||||
const runtime=path.join(root,path.basename(runtimeBefore)),cwd=path.join(root,path.basename(fixture.cwd));
|
||||
const replace=(value:string)=>value.replaceAll(runtimeBefore,runtime).replaceAll(fixture.cwd,cwd);
|
||||
const hook=JSON.parse(replace(JSON.stringify(fixture.hook))),stateRoot=replace(fixture.stateRoot),config=replace(fixture.config);
|
||||
const events=JSON.parse(replace(JSON.stringify(fixture.publicTools))) as NativePublicToolEvent[];
|
||||
const now=Date.parse(fixture.viewportCapturedAt),startedAt=Date.parse(fixture.commandStartedAt);
|
||||
const file=hook.pending.file,nativePlan=events.filter(e=>e.kind==='use'&&e.name==='Edit').at(-1)!.input!.file_path as string;
|
||||
for(const [target,content] of [[file,fixture.before],[nativePlan,fixture.nativePlanBefore]]) {
|
||||
fs.mkdirSync(path.dirname(target),{recursive:true});fs.writeFileSync(target,content);
|
||||
const at=new Date(Date.parse(hook.pending.timestamp)-1000);fs.utimesSync(target,at,at);
|
||||
}
|
||||
fs.mkdirSync(cwd,{recursive:true});fs.mkdirSync(path.dirname(hook.pending.transcriptPath),{recursive:true});
|
||||
const records=events.map(e=>({sessionId:e.sessionId,cwd,isSidechain:false,timestamp:e.timestamp,requestId:e.requestId,
|
||||
message:{id:e.messageId,role:e.kind==='use'?'assistant':'user',content:e.kind==='use'?[{type:'tool_use',id:e.toolUseId,name:e.name,input:e.input}]:[{type:'tool_result',tool_use_id:e.toolUseId,content:e.content??'',is_error:e.isError}]}}));
|
||||
fs.writeFileSync(hook.pending.transcriptPath,records.map(r=>JSON.stringify(r)).join('\n')+'\n');
|
||||
const hookFile=path.join(root,'hook.json');fs.writeFileSync(hookFile,JSON.stringify(hook)+'\n');
|
||||
const publicTools:NativePublicToolEvent[]=[];const transcript=readPlanCountTranscript(config,cwd,e=>publicTools.push(e));
|
||||
const pending=readPendingAutoplanArtifact(hookFile,cwd,config,stateRoot,startedAt,publicTools,now,true);
|
||||
const context={cwd,ownedStateRoot:stateRoot,ownedNativePlansRoot:path.join(config,'plans'),commandStartedAt:startedAt,
|
||||
now,viewportCapturedAt:now,transcriptStatus:transcript.status,publicTools,pending};
|
||||
const viewport=replace(fixture.viewport);
|
||||
const invoke=(screen=viewport,ctx=context,seen=new Set<string>())=>permission.publishedAutoplanArtifactPermissionInput(screen,ctx,seen);
|
||||
return {root,hook,hookFile,config,file,nativePlan,context,viewport,invoke,dispose:()=>fs.rmSync(root,{recursive:true,force:true})};
|
||||
}
|
||||
type Replay=ReturnType<typeof replay>;
|
||||
const current=(r:Replay)=>r.context.publicTools.find(e=>e.toolUseId===r.hook.pending.toolUseId&&e.kind==='use')!;
|
||||
const queued=(r:Replay)=>r.context.publicTools.filter(e=>e.kind==='use'&&e.name==='Edit'&&Date.parse(e.timestamp)>Date.parse(r.hook.pending.timestamp));
|
||||
function reject(cases:Array<[string,(r:Replay)=>void]>) {
|
||||
for(const [name,change] of cases){const r=replay();try{change(r);expect(r.invoke(),name).toBeNull()}finally{r.dispose()}}
|
||||
}
|
||||
|
||||
test('the retained pending CEO edit remains distinct from later published native-plan edits',()=>{
|
||||
const r=replay();try{
|
||||
expect(autoplanArtifactRecorderStatus(r.hookFile,r.context.cwd,r.config,r.context.ownedStateRoot)).toEqual({status:'pending'});
|
||||
expect(r.context.pending?.toolUseId).toBe(fixture.hook.pending.toolUseId);
|
||||
expect(queued(r)).toHaveLength(2);
|
||||
expect(permission.autoplanArtifactPermissionInput(r.viewport,r.context,new Set())).toBeNull();
|
||||
expect(permission.pendingAutoplanArtifactPermissionInput(r.viewport,r.context,new Set())).toBeNull();
|
||||
expect(r.invoke()).toEqual({input:'1\r',signature:fixture.hook.sessionId+':'+fixture.hook.pending.toolUseId,file:r.file});
|
||||
expect(fixture.provenance.retrospectivePass).toBe(false);
|
||||
expect(r.invoke(r.viewport,r.context,new Set([r.hook.sessionId+':'+r.hook.pending.toolUseId]))).toBeNull();
|
||||
expect(r.invoke(r.viewport,r.context,new Set([permission.autoplanArtifactMenuKey(r.viewport)]))).toBeNull();
|
||||
}finally{r.dispose()}
|
||||
});
|
||||
|
||||
test('a bare current panel and its bound redraw labels represent the same one-time permission',()=>{
|
||||
const r=replay();try{
|
||||
const title=r.viewport.indexOf('● Update('),panel=r.viewport.indexOf('────────────────');
|
||||
expect(r.invoke(r.viewport.slice(title))?.input).toBe('1\r');
|
||||
expect(r.invoke(r.viewport.slice(panel))?.input).toBe('1\r');
|
||||
}finally{r.dispose()}
|
||||
});
|
||||
|
||||
test('only unstarted same-batch publications to the launcher-owned native plans root may wait behind it',()=>{
|
||||
reject([
|
||||
['no launcher root',r=>{delete (r.context as any).ownedNativePlansRoot}],
|
||||
['foreign launcher root',r=>{r.context.ownedNativePlansRoot=path.join(r.root,'foreign')}],
|
||||
['foreign message',r=>{queued(r)[0]!.messageId='msg_other'}],
|
||||
['foreign request',r=>{queued(r)[0]!.requestId='req_other'}],
|
||||
['foreign session',r=>{queued(r)[0]!.sessionId='other'}],
|
||||
['foreign target',r=>{queued(r)[0]!.input!.file_path=r.file+'.other'}],
|
||||
['queued Write',r=>{queued(r)[0]!.name='Write'}],
|
||||
['replace-all successor',r=>{queued(r)[0]!.input!.replace_all=true}],
|
||||
['already started successor',r=>{r.context.pending!.hookSeenIds!.push(queued(r)[0]!.toolUseId)}],
|
||||
['successor completion',r=>{const q=queued(r)[0]!;r.context.publicTools.push({kind:'result',sessionId:q.sessionId,toolUseId:q.toolUseId,timestamp:new Date(r.context.now).toISOString(),isError:false})}],
|
||||
['successor failure',r=>{const q=queued(r)[0]!;r.context.publicTools.push({kind:'result',sessionId:q.sessionId,toolUseId:q.toolUseId,timestamp:new Date(r.context.now).toISOString(),isError:true})}],
|
||||
['successor published after viewport',r=>{queued(r)[0]!.timestamp=new Date(r.context.viewportCapturedAt+1).toISOString()}],
|
||||
['missing native plan',r=>{fs.unlinkSync(r.nativePlan)}],
|
||||
['native plan changed after current hook',r=>{fs.utimesSync(r.nativePlan,new Date(r.context.now),new Date(r.context.now))}],
|
||||
['symlink native plan',r=>{const other=path.join(r.root,'other.md');fs.renameSync(r.nativePlan,other);fs.symlinkSync(other,r.nativePlan)}],
|
||||
['successful Read cannot replace native-plan mutation history',r=>{for(const e of r.context.publicTools)if(e.kind==='use'&&e.input?.file_path===r.nativePlan&&Date.parse(e.timestamp)<Date.parse(r.hook.pending.timestamp))e.name='Read'}],
|
||||
['no successful native-plan history',r=>{const ids=new Set(r.context.publicTools.filter(e=>e.input?.file_path===r.nativePlan).map(e=>e.toolUseId));for(const e of r.context.publicTools)if(e.kind==='result'&&ids.has(e.toolUseId))e.isError=true}],
|
||||
]);
|
||||
});
|
||||
|
||||
test('the active hook, current digest, successful owned history and time remain mandatory',()=>{
|
||||
reject([
|
||||
['no current hook',r=>{r.context.pending=undefined}],['foreign hook',r=>{r.context.pending!.sessionId='other'}],
|
||||
['wrong current ID',r=>{r.context.pending!.toolUseId=queued(r)[0]!.toolUseId}],
|
||||
['no digest',r=>{delete r.context.pending!.editDigest}],
|
||||
['changed digest',r=>{r.context.pending!.editDigest!.requestSHA256='0'.repeat(64)}],
|
||||
['changed replacement',r=>{current(r).input!.new_string+=' changed'}],
|
||||
['changed current file',r=>{fs.appendFileSync(r.file,'changed');fs.utimesSync(r.file,new Date(0),new Date(0))}],
|
||||
['completed current',r=>{const q=current(r);r.context.publicTools.push({kind:'result',sessionId:q.sessionId,toolUseId:q.toolUseId,timestamp:new Date(r.context.now).toISOString(),isError:false})}],
|
||||
['current file newer than hook',r=>{fs.utimesSync(r.file,new Date(r.context.now),new Date(r.context.now))}],
|
||||
['pending after viewport',r=>{r.context.pending!.timestamp=new Date(r.context.now+1).toISOString()}],
|
||||
['stale hook',r=>{r.context.pending!.timestamp=new Date(r.context.commandStartedAt-1).toISOString()}],
|
||||
['unavailable transcript',r=>{r.context.transcriptStatus='missing'}],
|
||||
]);
|
||||
const r=replay();try{
|
||||
fs.writeFileSync(r.hookFile+'.invalid','{"reason":"concurrent_pending"}');
|
||||
expect(readPendingAutoplanArtifact(r.hookFile,r.context.cwd,r.config,r.context.ownedStateRoot,r.context.commandStartedAt,r.context.publicTools,r.context.now,true)).toBeUndefined();
|
||||
}finally{r.dispose()}
|
||||
});
|
||||
|
||||
test('completed output and redraw labels cannot hide a foreign, quoted or persistent-permission panel',()=>{
|
||||
const changes:Array<[string,(s:string)=>string]>=[
|
||||
['example prefix',s=>'Example:\n'+s],['quoted whole pane',s=>s.split('\n').map(r=>'> '+r).join('\n')],
|
||||
['arbitrary output',s=>s.replace('"changed": true','"changed": false')],
|
||||
['foreign completed command',s=>s.replace('with-skills/.clau','foreign/.clau')],
|
||||
['missing one redraw',s=>s.replace('● Updated plan','')],['extra redraw',s=>s.replace('● Updated plan','● Updated plan\n● Updated plan')],
|
||||
['arbitrary redraw prose',s=>s.replace('● Updated plan','● Example plan')],
|
||||
['foreign current title',s=>s.replace('Update(~/.gstack/','Update(/foreign/')],
|
||||
['foreign displayed project',s=>s.replace('…-207152-jk89F3/skill-home-bOPSw5/.gstack/projects/gstack-autoplan-chain-kVh2Sb','…projects/foreign')],
|
||||
['different requested addition',s=>s.replace('## Reviewer Concerns','## An unrelated edit')],
|
||||
['wrong menu file',s=>s.replace('user-dashboard.md?','other.md?')],
|
||||
['persistent session approval',s=>s.replace('❯ 1. Yes','❯ 2. Yes')],['trailing prose',s=>s+'\nAnother prompt'],
|
||||
];
|
||||
for(const [name,edit] of changes){const r=replay();try{expect(r.invoke(edit(r.viewport)),name).toBeNull()}finally{r.dispose()}}
|
||||
});
|
||||
|
||||
test('only Autoplan discovers the permission regression and its captured fixture',()=>{
|
||||
for(const file of ['test/autoplan-artifact-stall-as.test.ts','test/fixtures/autoplan-artifact-stall-as.json'])
|
||||
expect(selectTests([file],E2E_TOUCHFILES,[]).selected).toEqual(['autoplan-chain-pty']);
|
||||
});
|
||||
@@ -0,0 +1,85 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
import captured from './fixtures/coverage-diagram-legend-as.json';
|
||||
import billing from './fixtures/coverage-audit-ae.json';
|
||||
import { coverageAuditVerdict } from './helpers/coverage-audit-evidence';
|
||||
import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles';
|
||||
|
||||
function verdict(output: string, index = 0) {
|
||||
const row = captured.rows[index]!;
|
||||
return coverageAuditVerdict({ ...row.result, output } as any, {
|
||||
cwd: row.cwd,
|
||||
source: { path: row.cwd + '/src/billing.ts', content: billing.files.source },
|
||||
tests: { path: row.cwd + '/test/billing.test.ts', content: billing.files.tests },
|
||||
});
|
||||
}
|
||||
const diagram = (output: string) => verdict(output).diagram;
|
||||
const flat = (legend = 'Legend: [✔] tested [✘] GAP (no test)') => '```text\n' + legend + '\nprocessPayment(amount, currency)\n├──► return success [✔]\nrefundPayment(paymentId, reason)\n└──► return refunded [✘]\n```';
|
||||
|
||||
test('both exact public outputs contain the seeded diagram and retain actual native file delivery', () => {
|
||||
for (let i = 0; i < captured.rows.length; i++) {
|
||||
expect(verdict(captured.rows[i]!.result.output, i)).toEqual({ sourceRead: true, testsRead: true, diagram: true, passed: true, failures: [] });
|
||||
}
|
||||
expect(captured.provenance.originalAttemptOutcomes).toEqual(['failed', 'failed']);
|
||||
expect(captured.provenance.paidOutcomesReclassified).toBe(false);
|
||||
});
|
||||
|
||||
test('closed legend annotations preserve the same two meanings and arrow branch ownership', () => {
|
||||
for (const legend of ['Legend: [✔] tested [✘] GAP', 'Legend: [✔] tested [✘] GAP (no test)', 'Legend: [✔] tested [✘] GAP ──► branch', 'Legend: [✔] tested [✘] GAP (no test) ──► branch']) {
|
||||
expect(diagram(flat(legend))).toBe(true);
|
||||
expect(diagram(flat(legend).replace(/^([├└]─+)►/gm, '$1'))).toBe(true);
|
||||
expect(diagram(flat(legend).replaceAll('✔', '✓').replaceAll('✘', '✗'))).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('extra legend explanations cannot invert, qualify or fabricate coverage meanings', () => {
|
||||
for (const legend of ['', 'Legend: [✔] GAP [✘] tested', 'Legend: [✔] tested [✘] tested', 'Legend: [✔] tested [✘] GAP (not a gap)', 'Legend: [✔] tested [✘] GAP except refunds', 'Legend: [✔] tested [✘] GAP [✘] covered', 'Example: [✔] tested [✘] GAP', 'Legend: not [✔] tested [✘] GAP', 'Legend: [✔] tested [✘] GAP ──► covered']) {
|
||||
expect(diagram(flat(legend))).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('a branch status correction supplies its final state and ambiguous markers supply neither', () => {
|
||||
expect(diagram(flat().replace('return refunded [✘]', 'return refunded [✔]→[✘]'))).toBe(true);
|
||||
expect(diagram(flat().replace('return success [✔]', 'return success [✘]->[✔]'))).toBe(true);
|
||||
expect(diagram(flat().replace('return success [✔]', 'return success [✔]→[✘]'))).toBe(false);
|
||||
expect(diagram(flat().replace('return refunded [✘]', 'return refunded [✘]→[✔]'))).toBe(false);
|
||||
expect(diagram(flat().replace('return success [✔]', 'return success [✔] [✘]'))).toBe(false);
|
||||
expect(diagram(flat().replace('return refunded [✘]', 'return refunded [✘] [✔]'))).toBe(false);
|
||||
});
|
||||
|
||||
test('literal labels cannot override a final or ambiguous bracketed symbol state', () => {
|
||||
for (const [old, replacement] of [
|
||||
['return success [✔]', 'return success TESTED [✔]→[✘]'],
|
||||
['return refunded [✘]', 'return refunded UNTESTED [✘]→[✔]'],
|
||||
['return success [✔]', 'return success TESTED [✔] [✘]'],
|
||||
['return refunded [✘]', 'return refunded [GAP] [✘] [✔]'],
|
||||
]) expect(diagram(flat().replace(old!, replacement!))).toBe(false);
|
||||
});
|
||||
|
||||
test('each seeded function must own its own branch and legend in the same current diagram', () => {
|
||||
for (const output of [
|
||||
flat().replace('refundPayment', 'otherRefund'), flat().replace('processPayment', 'otherPayment'),
|
||||
flat().replace('├──► return success [✔]', 'unrelatedHelper()\n├──► return success [✔]'),
|
||||
flat().replace('└──► return refunded [✘]', 'unrelatedHelper()\n└──► return refunded [✘]'),
|
||||
flat().split('\n').map(line => '> ' + line).join('\n'), '````markdown\n' + flat() + '\n````',
|
||||
'Example:\n' + flat(), flat().replace('[✔] tested [✘] GAP (no test)', '[✔] tested (not covered) [✘] GAP'),
|
||||
'```text\nLegend: [✔] tested [✘] GAP\n```\n' + flat(''),
|
||||
]) expect(diagram(output)).toBe(false);
|
||||
});
|
||||
|
||||
test('successful diagram parsing cannot replace successful capture or native file delivery', () => {
|
||||
const row = captured.rows[0]!;
|
||||
const files = { cwd: row.cwd, source: { path: row.cwd + '/src/billing.ts', content: billing.files.source }, tests: { path: row.cwd + '/test/billing.test.ts', content: billing.files.tests } };
|
||||
for (const mutate of [
|
||||
(r: any) => { r.exitReason = 'timeout'; }, (r: any) => { r.browseErrors = ['read failed']; },
|
||||
(r: any) => { r.transcript = []; }, (r: any) => { r.transcript[2].message.content[0].is_error = true; },
|
||||
(r: any) => { r.transcript[2].message.content[0].content = 'Both filenames were read'; },
|
||||
]) {
|
||||
const result = structuredClone(row.result); mutate(result);
|
||||
const checked = coverageAuditVerdict(result as any, files);
|
||||
expect(checked.diagram).toBe(true); expect(checked.passed).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('new parser regression artifacts select both coverage audit owners', () => {
|
||||
for (const file of ['test/coverage-diagram-legend-as.test.ts', 'test/fixtures/coverage-diagram-legend-as.json']) expect(selectTests([file], E2E_TOUCHFILES, []).selected.sort()).toEqual(['plan-eng-coverage-audit', 'review-coverage-audit']);
|
||||
});
|
||||
@@ -0,0 +1,138 @@
|
||||
import {describe,expect,test} from 'bun:test';
|
||||
import {nativePlanCallFingerprint,planCountQuestionPhase,designStep0Boundary} from './helpers/claude-pty-runner';
|
||||
import {isDesignCountFirstReview,isDesignCountSetup,isDesignCompletionHandoff} from './helpers/design-count-review';
|
||||
import {isDesignArtifactGeneration} from './helpers/design-artifact-question';
|
||||
import type {NativePlanQuestionCall} from './helpers/plan-count-transcript';
|
||||
import captured from './fixtures/design-primary-group-as-calls.json';
|
||||
|
||||
const calls=()=>structuredClone(captured.calls) as NativePlanQuestionCall[];
|
||||
const first=()=>calls()[1]!;
|
||||
const fp=(c:NativePlanQuestionCall)=>nativePlanCallFingerprint(c,0,true);
|
||||
const reanswer=(c:NativePlanQuestionCall)=>{c.answers={[c.questions[0]!.question]:c.questions[0]!.options[0]!.label};return c;};
|
||||
const accepted=(c:NativePlanQuestionCall)=>isDesignCountFirstReview(fp(c));
|
||||
|
||||
describe('Design primary action named by the Issue header',()=>{
|
||||
test('exact eight native calls retain the three setup questions in one call and the six issues plus TODO',()=>{
|
||||
const input=calls(), before=JSON.stringify(input);
|
||||
expect(input.map(c=>c.questions.length)).toEqual([3,1,1,1,1,1,1,1]);
|
||||
expect(input.flatMap(c=>c.questions)).toHaveLength(10);
|
||||
let started=false; const phases=input.map(call=>{
|
||||
const phase=planCountQuestionPhase(fp(call),started,designStep0Boundary,isDesignCountFirstReview,isDesignCountSetup,isDesignCompletionHandoff,isDesignArtifactGeneration);
|
||||
started=phase.reviewStarted;return phase;
|
||||
});
|
||||
expect(phases.map(p=>p.preReview)).toEqual([true,false,false,false,false,false,false,false]);
|
||||
expect(phases.filter(p=>p.administrative)).toHaveLength(0);
|
||||
expect(input.map(accepted)).toEqual([false,true,false,false,false,false,false,false]);
|
||||
expect(phases.filter(p=>!p.preReview)).toHaveLength(7);
|
||||
expect(phases.filter(p=>p.preReview)).toHaveLength(1);
|
||||
expect(JSON.stringify(input)).toBe(before);
|
||||
});
|
||||
test('finding annotations, control names, palette and option order do not supply or restrict identity',()=>{
|
||||
for(const annotation of [' (F1)',' (F27)','']){
|
||||
const c=first(),q=c.questions[0]!;
|
||||
q.question=q.question.replace(' (F1)',annotation);
|
||||
expect(accepted(reanswer(c))).toBe(true);
|
||||
}
|
||||
const c=JSON.parse(JSON.stringify(first()).replaceAll('Save','Publish').replaceAll('#1d4ed8','#123abc')) as NativePlanQuestionCall;
|
||||
const q=c.questions[0]!;
|
||||
q.header='Issue 8: Publish';
|
||||
q.question=q.question.replace('D4 — Issue 1 (F1)','D31 — Issue 8 (F12)').replace(/\b1([ABC])\b/g,'8$1');
|
||||
for(const o of q.options)o.label=o.label.replace(/^1/,'8');
|
||||
q.options.reverse();
|
||||
for(const o of q.options){c.answers={[q.question]:o.label};expect(accepted(c)).toBe(true);}
|
||||
q.options.reverse();q.options=q.options.filter(o=>!o.label.startsWith('8B:'));expect(accepted(reanswer(c))).toBe(true);
|
||||
});
|
||||
test('the separately completed retry retains its existing two setup and six review calls',()=>{
|
||||
const input=structuredClone(captured.retryCalls) as NativePlanQuestionCall[];
|
||||
let started=false;const phases=input.map(call=>{
|
||||
const phase=planCountQuestionPhase(fp(call),started,designStep0Boundary,isDesignCountFirstReview,isDesignCountSetup,isDesignCompletionHandoff,isDesignArtifactGeneration);
|
||||
started=phase.reviewStarted;return phase;
|
||||
});
|
||||
expect(input).toHaveLength(8);
|
||||
expect(phases.map(p=>p.preReview)).toEqual([true,true,false,false,false,false,false,false]);
|
||||
expect(phases.filter(p=>p.administrative)).toHaveLength(0);
|
||||
});
|
||||
test('primary and ghost controls, documented count, issue identity and offered choice identities remain bound',()=>{
|
||||
const changes:Array<(c:NativePlanQuestionCall)=>void>=[
|
||||
c=>{c.questions[0]!.header='Issue 1';},
|
||||
c=>{c.questions[0]!.header='Issue 1: Export';},
|
||||
c=>{c.questions[0]!.header='Issue 2: Save';},
|
||||
c=>{c.questions[0]!.question=c.questions[0]!.question.replace('other three','other two');},
|
||||
c=>{c.questions[0]!.question=c.questions[0]!.question.replace('Reset, Cancel and Export','Reset, Reset and Export');},
|
||||
c=>{c.questions[0]!.question=c.questions[0]!.question.replace('Reset, Cancel and Export','Reset, Save and Export');},
|
||||
c=>{c.questions[0]!.options[0]!.description=c.questions[0]!.options[0]!.description!.replace('Reset, Cancel, Export','Reset, Cancel, Delete');},
|
||||
c=>{c.questions[0]!.options[0]!.description=c.questions[0]!.options[0]!.description!.replace('Reset, Cancel, Export','Reset, Cancel, Save');},
|
||||
c=>{c.questions[0]!.options[0]!.description=c.questions[0]!.options[0]!.description!.replace('Reset, Cancel, Export','Reset, Cancel, Export, Export');},
|
||||
c=>{c.questions[0]!.options[2]!.label='1C: Keep all three identical';},
|
||||
c=>{c.questions[0]!.options[2]!.label='2C: Keep all four identical';},
|
||||
];
|
||||
for(const change of changes){const c=first();change(c);expect(accepted(reanswer(c))).toBe(false);}
|
||||
});
|
||||
test('readiness, focus, navigation, source-only questions and naked F labels cannot begin review',()=>{
|
||||
for(const title of [
|
||||
'D4 — Issue 1 (F1): Ready to review the header action group?',
|
||||
'D4 — Issue 1 (F1): Which design source should the reviewer use?',
|
||||
'D4 — Issue 1 (F1): Fix the primary action?',
|
||||
'D4 — Issue 1 (F1): How should the header action group establish the primary action? Ready?',
|
||||
'Example: D4 — Issue 1 (F1): How should the header action group establish the primary action?',
|
||||
'> D4 — Issue 1 (F1): How should the header action group establish the primary action?',
|
||||
]){const c=first(),q=c.questions[0]!;q.question=title+'\n'+q.question.split('\n').slice(1).join('\n');expect(accepted(reanswer(c))).toBe(false);}
|
||||
for(const header of ['Focus','Routing','Next steps','Outside voices']){const c=first();c.questions[0]!.header=header;expect(accepted(c)).toBe(false);}
|
||||
const c=first();c.questions[0]!.options=[{label:'Start the review'},{label:'Wait'}];expect(accepted(reanswer(c))).toBe(false);
|
||||
});
|
||||
test('current gap and contract cannot be replaced by quoted, historical or conditional material',()=>{
|
||||
for(const prefix of ['Historical example: ','Hypothetical example: ','Quoted assessment: ','Source example: ','If approved, ','When approved, ','Unless rejected, ','Assuming approval, ','Provided approval, ']){
|
||||
const c=first();c.questions[0]!.question=c.questions[0]!.question.replace('ELI10: ','ELI10: '+prefix);expect(accepted(reanswer(c))).toBe(false);
|
||||
}
|
||||
for(const transform of [(s:string)=>'"'+s+'"',(s:string)=>'> '+s,(s:string)=>' '+s,(s:string)=>'```\n'+s+'\n```']){
|
||||
const c=first(),q=c.questions[0]!;q.question=q.question.split('\n').map(line=>line.startsWith('ELI10:')?transform(line):line).join('\n');expect(accepted(reanswer(c))).toBe(false);
|
||||
}
|
||||
for(const suffix of [' This finding is no longer current.',' This finding is "no longer current".',' This requirement is withdrawn.',' This contract is "withdrawn".',' This gap is now resolved.',' This issue is superseded.']){
|
||||
const c=first();c.questions[0]!.question+=suffix;expect(accepted(reanswer(c))).toBe(false);
|
||||
}
|
||||
});
|
||||
test('each offered amendment and deferral must remain current and unconditional',()=>{
|
||||
for(const index of [0,2])for(const prefix of ['Historical example: ','Source example: ','Assuming approval, ','Provided approval, ','✅ Assuming approval, ','✅ Provided approval, ']){
|
||||
const c=first(),o=c.questions[0]!.options[index]!;o.description=prefix+o.description;expect(accepted(reanswer(c))).toBe(false);
|
||||
}
|
||||
for(const index of [0,2])for(const suffix of [' This finding is no longer current.',' This amendment is "withdrawn".',' This deferral is rejected.',' This choice is superseded.',' This gap is closed.',' This contract is withdrawn.',' This requirement is "no longer current".',' Assuming approval, this is proposed only.',' Provided approval, this will become current.']){
|
||||
const c=first();c.questions[0]!.options[index]!.description+=suffix;expect(accepted(reanswer(c))).toBe(false);
|
||||
}
|
||||
for(const suffix of [' These tokens are withdrawn.',' These styles are "no longer current".',' Do not apply these tokens.']){
|
||||
const c=first();c.questions[0]!.options[0]!.description+=suffix;expect(accepted(reanswer(c))).toBe(false);
|
||||
}
|
||||
const c=first();c.questions[0]!.options[2]!.description+=' Do not keep all four buttons identical.';expect(accepted(reanswer(c))).toBe(false);
|
||||
});
|
||||
test('quoted past statuses do not erase the current finding, style or opposed choice',()=>{
|
||||
for(const target of [-1,0,2])for(const history of [' The prior review said "This finding is no longer current."'," The prior review said 'This finding is withdrawn.'",' The prior review said ‘This finding is no longer current.’',' The prior review said "Estimate (human: ~1h / CC: ~5min) This finding is no longer current."',' The prior review said `This finding is no longer current.`',' The earlier decision was `no longer current`.','\n> This amendment is withdrawn.']){
|
||||
const c=first();if(target<0)c.questions[0]!.question+=history;else c.questions[0]!.options[target]!.description+=history;
|
||||
expect(accepted(reanswer(c))).toBe(true);
|
||||
}
|
||||
});
|
||||
test('current status scalars retain their subjects across quote styles and semicolon boundaries',()=>{
|
||||
for(const target of [-1,0,2])for(const subject of ['finding','amendment','contract'])for(const status of ['withdrawn','no longer current'])for(const quote of ['',"'","‘",'"','“','`'])for(const boundary of [' ','; ']){
|
||||
const closing=quote==='‘'?'’':quote==='“'?'”':quote;
|
||||
const suffix=boundary+'This '+subject+' is '+quote+status+closing+'.';
|
||||
const c=first();if(target<0)c.questions[0]!.question+=suffix;else c.questions[0]!.options[target]!.description+=suffix;
|
||||
expect(accepted(reanswer(c))).toBe(false);
|
||||
}
|
||||
});
|
||||
test('completed native ownership, answer membership, one question and exact option indices are required',()=>{
|
||||
const changes:Array<(c:NativePlanQuestionCall)=>void>=[
|
||||
c=>{c.answered=false;},c=>{delete (c as Partial<NativePlanQuestionCall>).answered;},
|
||||
c=>{c.failed=true;},c=>{delete c.failed;},c=>{c.sessionId='';},c=>{c.toolUseId='';},
|
||||
c=>{c.answers={};},c=>{c.answers={[c.questions[0]!.question]:'not offered'};},
|
||||
c=>{delete c.answeredAt;},c=>{c.answeredAt='invalid';},c=>{delete c.unansweredQuestionIndices;},c=>{c.unansweredQuestionIndices=[0];},
|
||||
c=>{c.questions[0]!.multiSelect=true;},c=>{c.questions.push(structuredClone(c.questions[0]!));},
|
||||
c=>{c.questions[0]!.options.push(structuredClone(c.questions[0]!.options[0]!));},
|
||||
];
|
||||
for(const change of changes){const c=first();change(c);expect(accepted(c)).toBe(false);}
|
||||
for(const mutate of [
|
||||
(f:ReturnType<typeof fp>)=>{f.signature='foreign';},
|
||||
(f:ReturnType<typeof fp>)=>{f.nativeQuestionIndex=1;},
|
||||
(f:ReturnType<typeof fp>)=>{f.options=[];},
|
||||
(f:ReturnType<typeof fp>)=>{f.options[0]!.index=2;},
|
||||
(f:ReturnType<typeof fp>)=>{f.options[0]!.label='unrelated';},
|
||||
]){const f=fp(first());mutate(f);expect(isDesignCountFirstReview(f)).toBe(false);}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,179 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import fixture from './fixtures/dx-asserted-defect-as.json';
|
||||
import retryFixture from './fixtures/dx-asserted-defect-as-retry.json';
|
||||
import { devexSeedCoverage } from './helpers/devex-seed-coverage';
|
||||
import type { NativePlanQuestionCall, PlanCountTranscript } from './helpers/plan-count-transcript';
|
||||
import { E2E_TOUCHFILES, matchGlob } from './helpers/touchfiles';
|
||||
|
||||
const missed = [1, 2, 4];
|
||||
function transcript(): PlanCountTranscript {
|
||||
return { status: 'ready', calls: structuredClone(fixture.calls) as NativePlanQuestionCall[], assistantMessages: [] };
|
||||
}
|
||||
function retryTranscript(): PlanCountTranscript {
|
||||
return { status: 'ready', calls: structuredClone(retryFixture.calls) as NativePlanQuestionCall[], assistantMessages: [] };
|
||||
}
|
||||
function change(t: PlanCountTranscript, i: number, edit: (s: string) => string) {
|
||||
const c = t.calls[i]!, q = c.questions[0]!, answer = c.answers![q.question]!;
|
||||
q.question = edit(q.question); c.answers = { [q.question]: answer };
|
||||
}
|
||||
function title(t: PlanCountTranscript, i: number, edit: (s: string) => string) {
|
||||
change(t, i, s => { const lines = s.split('\n'); lines[0] = edit(lines[0]!); return lines.join('\n'); });
|
||||
}
|
||||
function replaceTitle(t: PlanCountTranscript, i: number, s: string) { title(t, i, () => `D${i} — ${s}`); }
|
||||
function absentStage(t: PlanCountTranscript) {
|
||||
replaceTitle(t, 4, "Journey stage INSTALL / HELLO WORLD: the quickstart's first command points at a file that does not ship.");
|
||||
}
|
||||
|
||||
describe('DX asserted defect heading families', () => {
|
||||
test('the exact completed first attempt has five distinct decisions without changing historical outcomes', () => {
|
||||
const t = transcript(), before = JSON.stringify(t), result = devexSeedCoverage(t);
|
||||
expect(t.calls).toHaveLength(6); expect(result.complete).toBe(true); expect(result.missing).toEqual([]);
|
||||
expect(Object.values(result.decisions).flat().sort()).toEqual(t.calls.slice(1).map(c => `${c.sessionId}:${c.toolUseId}`).sort());
|
||||
expect(JSON.stringify(t)).toBe(before); expect(fixture.provenance.paidOutcomesReclassified).toBe(false);
|
||||
expect(fixture.provenance.historicalOutcome).toContain('seed predicates failed');
|
||||
for (let i = 1; i <= 5; i++) { const copy = transcript(); copy.calls.splice(i, 1); expect(devexSeedCoverage(copy).missing).toHaveLength(1); }
|
||||
});
|
||||
test('equivalent nominal prerequisites and explicit signature comparisons retain concrete alternatives', () => {
|
||||
for (const heading of ['Required remote CI gate before the first local evaluation', 'Mandatory 30-second CI check before first local run.', 'Mandatory CI check before the first local result?']) {
|
||||
const t = transcript(); replaceTitle(t, 1, heading); expect(devexSeedCoverage(t).complete).toBe(true);
|
||||
}
|
||||
for (const heading of ['`run_eval(dataset, evaluator)` versus `run_batch(evaluator, dataset)`: opposite argument order', 'run_eval(dataset, evaluator) vs. run_batch(evaluator, dataset): swapped positional order?', 'run_eval(dataset, evaluator) and run_batch(evaluator, dataset): reversed positional order.']) {
|
||||
const t = transcript(); replaceTitle(t, 2, heading); expect(devexSeedCoverage(t).complete).toBe(true);
|
||||
}
|
||||
for (const i of missed) { const t = transcript(), c = t.calls[i]!, q = c.questions[0]!;
|
||||
for (const option of q.options) { c.answers = { [q.question]: option.label }; expect(devexSeedCoverage(t).complete).toBe(true); }
|
||||
}
|
||||
});
|
||||
test('same-file absence and source-defined stage vocabulary normalize without paid retry credit', () => {
|
||||
for (const suffix of ['not in the wheel or the release examples archive', 'not in the package', 'not in the wheel?']) {
|
||||
const t = transcript(); title(t, 4, s => s.replace('not in the package or the examples archive', suffix)); expect(devexSeedCoverage(t).complete).toBe(true);
|
||||
}
|
||||
// Synthetic title controls based on the source-defined journey vocabulary.
|
||||
// The fixture above contains only completed first-attempt native calls.
|
||||
for (const stage of ['INSTALL / HELLO WORLD', 'Install', 'DISCOVER / INSTALL', 'HELLO WORLD', 'REAL USAGE', 'DEBUG', 'UPGRADE']) {
|
||||
const t = transcript(); absentStage(t); title(t, 4, s => s.replace('INSTALL / HELLO WORLD', stage)); expect(devexSeedCoverage(t).complete).toBe(true);
|
||||
}
|
||||
for (const stage of ['SOURCE / HELLO WORLD', 'INSTALL / ARCHIVE', 'OLD INSTALL', 'DEPLOYMENT']) {
|
||||
const t = transcript(); absentStage(t); title(t, 4, s => s.replace('INSTALL / HELLO WORLD', stage)); expect(devexSeedCoverage(t).complete).toBe(false);
|
||||
}
|
||||
});
|
||||
test('healthy, optional, foreign and unasserted headings cannot borrow repairs from their options', () => {
|
||||
for (const [i, heading] of [
|
||||
[1, 'Optional remote CI check before the first local result'], [1, 'Mandatory remote CI check after the first local result'],
|
||||
[1, 'Current issue'], [2, 'run_eval(dataset, evaluator) vs run_batch(dataset, evaluator): same positional order'],
|
||||
[2, 'run_score(dataset, evaluator) vs run_batch(evaluator, dataset): reversed positional order'], [2, 'Function signatures'],
|
||||
[4, 'README quickstart points at examples/first_eval.py, which is in the package and the examples archive'],
|
||||
[4, 'README quickstart does not point at a file that does not ship'],
|
||||
[4, 'README quickstart points at a file that does ship'],
|
||||
] as const) for (const punctuation of ['', '?']) { const t = transcript(); replaceTitle(t, i, heading + punctuation); expect(devexSeedCoverage(t).complete, `${i}: ${heading}${punctuation}`).toBe(false); }
|
||||
});
|
||||
test('punctuation never bypasses title, metadata or explanation ownership', () => {
|
||||
for (const questionMark of ['', '?']) for (const i of missed) {
|
||||
for (const prefix of ['Source: ', 'Source. ', 'Historical example: ', 'Earlier review: ', 'Quoted source: ', 'If approved, ', 'Assuming approval, ', 'Provided approval, ', '> ', '"', '`']) {
|
||||
const t = transcript(); title(t, i, s => s.replace(/^(D\d+ — )(.*)$/, (_, id, body) => `${id}${prefix}${body}${prefix === '"' || prefix === '`' ? prefix : ''}${questionMark}`));
|
||||
expect(devexSeedCoverage(t).complete, `title ${i} ${prefix} ${questionMark}`).toBe(false);
|
||||
}
|
||||
for (const tail of [' if approved', ' once approved', ' after approval', ' pending approval']) {
|
||||
const t = transcript(); title(t, i, s => s + tail + questionMark); expect(devexSeedCoverage(t).complete).toBe(false);
|
||||
}
|
||||
for (const prefix of ['Source: ', 'Source. ', 'Historical assessment: ', 'If approved, ', 'Assuming approval, ', 'Provided approval, ']) {
|
||||
const t = transcript(); title(t, i, s => s + questionMark); change(t, i, s => s.replace('Project/branch/task: ', `Project/branch/task: ${prefix}`)); expect(devexSeedCoverage(t).complete).toBe(false);
|
||||
}
|
||||
for (const prefix of ['Source.\n', 'Hypothetical scenario.\n', '~~~\n', 'Earlier review assessment:\n']) {
|
||||
const t = transcript(); title(t, i, s => s + questionMark); change(t, i, s => s.replace('\nELI10:', `\n${prefix}ELI10:`)); expect(devexSeedCoverage(t).complete).toBe(false);
|
||||
}
|
||||
}
|
||||
});
|
||||
test('a current same-decision withdrawal wins; foreign, quoted and prospective statuses do not', () => {
|
||||
for (const i of missed) for (const punctuation of ['', '?']) {
|
||||
for (const tail of [`D${i} is withdrawn.`, `D ${i} is "superseded".`, 'This finding is cancelled.', 'This issue is "not current".', 'This finding is no longer current.', 'This finding is "no longer current".', "This finding is 'no longer current'.", 'This finding is `no longer current`.', `D${i} is "no longer current".`]) {
|
||||
const t = transcript(); title(t, i, s => s + punctuation); change(t, i, s => `${s}\n${tail}`); expect(devexSeedCoverage(t).complete).toBe(false);
|
||||
}
|
||||
for (const tail of ['D29 is withdrawn.', `> D${i} is withdrawn.`, `Earlier note: "D${i} is withdrawn."`, '```\nThis finding is cancelled.\n```', 'If the repair is accepted, this finding is resolved in the proposed API.']) {
|
||||
const t = transcript(); title(t, i, s => s + punctuation); change(t, i, s => `${s}\n${tail}`); expect(devexSeedCoverage(t).complete).toBe(true);
|
||||
}
|
||||
}
|
||||
});
|
||||
test('current offered remedies are required for the nominal and absence forms', () => {
|
||||
for (const i of missed) for (const mode of ['quoted', 'source', 'conditional', 'withdrawn', 'own-decision', 'navigation']) {
|
||||
const t = transcript(), c = t.calls[i]!, q = c.questions[0]!;
|
||||
q.options = q.options.map((o, n) => {
|
||||
if (mode === 'quoted') return { label: `"${o.label}" ${n}`, description: `"${o.description}"` };
|
||||
if (mode === 'source') return { label: `Reference ${n}`, description: `Source. ${o.label}\n${o.description}` };
|
||||
if (mode === 'conditional') return { label: `Alternative ${n}`, description: `Assuming approval, ${o.label}\n${o.description}` };
|
||||
if (mode === 'navigation') return { label: `Continue ${n}`, description: 'Move to the next section.' };
|
||||
return { ...o, description: `${o.description}\n${mode === 'own-decision' ? `D${i}` : 'This option'} is "withdrawn".` };
|
||||
});
|
||||
c.answers = { [q.question]: q.options[0]!.label }; expect(devexSeedCoverage(t).complete).toBe(false);
|
||||
}
|
||||
});
|
||||
test('owned option statuses remain binding after a line break or an effort estimate', () => {
|
||||
for (const i of missed) for (const separator of ['\n', ' ']) {
|
||||
for (const status of ['withdrawn', 'not current', 'no longer current']) for (const [open, close] of [['', ''], ["'", "'"], ['"', '"'], ['‘', '’'], ['“', '”'], ['`', '`']]) {
|
||||
const t = transcript(), q = t.calls[i]!.questions[0]!;
|
||||
for (const option of q.options) option.description += `${separator}This option is ${open}${status}${close}.`;
|
||||
expect(devexSeedCoverage(t).complete, `${i}: ${JSON.stringify(separator)} ${open}${status}${close}`).toBe(false);
|
||||
}
|
||||
for (const tail of ['Correction: This action is "no longer current".', `D${i} is 'not current'.`]) {
|
||||
const t = transcript(); for (const option of t.calls[i]!.questions[0]!.options) option.description += separator + tail;
|
||||
expect(devexSeedCoverage(t).complete).toBe(false);
|
||||
}
|
||||
for (const tail of ['D29 is "no longer current".', 'Old note: "This option is no longer current."', '"Archived proposal (human: ~1 day / CC: ~20 min) This option is withdrawn."', 'If the repair is accepted, this option is no longer current.']) {
|
||||
const t = transcript(); for (const option of t.calls[i]!.questions[0]!.options) option.description += separator + tail;
|
||||
expect(devexSeedCoverage(t).complete, `${i}: ${tail}`).toBe(true);
|
||||
}
|
||||
}
|
||||
});
|
||||
test('native completion, exact answer, distinct calls and session identity remain mandatory', () => {
|
||||
for (const change of [
|
||||
(t: PlanCountTranscript) => { t.status = 'missing'; },
|
||||
(t: PlanCountTranscript) => { t.calls[1]!.answered = false; },
|
||||
(t: PlanCountTranscript) => { t.calls[1]!.failed = true; },
|
||||
(t: PlanCountTranscript) => { t.calls[1]!.answeredAt = 'unknown'; },
|
||||
(t: PlanCountTranscript) => { t.calls[1]!.unansweredQuestionIndices = [0]; },
|
||||
(t: PlanCountTranscript) => { t.calls[1]!.answers = { foreign: 'Remove gate from local runs and demo (recommended)' }; },
|
||||
(t: PlanCountTranscript) => { t.calls[1]!.sessionId = 'foreign'; },
|
||||
(t: PlanCountTranscript) => { t.calls.push(structuredClone(t.calls[1]!)); },
|
||||
(t: PlanCountTranscript) => { t.calls[1]!.questions[0]!.multiSelect = true; },
|
||||
(t: PlanCountTranscript) => { t.calls[1]!.questions.push(structuredClone(t.calls[2]!.questions[0]!)); },
|
||||
]) { const t = transcript(); change(t); expect(devexSeedCoverage(t).complete).toBe(false); }
|
||||
});
|
||||
test('the separately completed retry retains its five exact current seed decisions and failed outcome', () => {
|
||||
const t = retryTranscript(), before = JSON.stringify(t), result = devexSeedCoverage(t);
|
||||
expect(t.calls).toHaveLength(14); expect(result.complete).toBe(true); expect(result.missing).toEqual([]);
|
||||
expect(Object.values(result.decisions).flat().sort()).toEqual([3, 4, 5, 6, 7, 10].map(i => `${t.calls[i]!.sessionId}:${t.calls[i]!.toolUseId}`).sort());
|
||||
expect(JSON.stringify(t)).toBe(before); expect(retryFixture.provenance.paidOutcomesReclassified).toBe(false);
|
||||
expect(retryFixture.provenance.historicalOutcome).toContain('seed predicates failed');
|
||||
});
|
||||
test('retry signature spelling and coded-error action evidence stay current and owned', () => {
|
||||
for (const i of [3, 5, 6]) {
|
||||
for (const tail of ['This finding is no longer current.', 'This finding is "no longer current".', "This finding is 'no longer current'.", 'This finding is `no longer current`.', `D${i + 1} is withdrawn.`]) {
|
||||
const t = retryTranscript(); change(t, i, s => `${s}\n${tail}`); expect(devexSeedCoverage(t).complete).toBe(false);
|
||||
}
|
||||
for (const prefix of ['Source: ', 'If approved, ', 'Assuming approval, ']) {
|
||||
const t = retryTranscript(); change(t, i, s => s.replace('ELI10: ', `ELI10: ${prefix}`)); expect(devexSeedCoverage(t).complete).toBe(false);
|
||||
}
|
||||
for (const tail of [`> D${i + 1} is withdrawn.`, `Old note: "D${i + 1} is no longer current."`, 'D39 is withdrawn.']) {
|
||||
const t = retryTranscript(); change(t, i, s => `${s}\n${tail}`); expect(devexSeedCoverage(t).complete).toBe(true);
|
||||
}
|
||||
}
|
||||
for (const text of ['take the same two concepts in the same positional order.', 'take the same two concepts in opposite positional order if approved.']) {
|
||||
const t = retryTranscript(); title(t, 5, s => s.replace('take the same two concepts in opposite positional order.', text)); expect(devexSeedCoverage(t).complete).toBe(false);
|
||||
}
|
||||
const healthy = retryTranscript(); title(healthy, 5, s => s.replace('run_batch(evaluator, dataset)', 'run_batch(dataset, evaluator)')); expect(devexSeedCoverage(healthy).complete).toBe(false);
|
||||
for (const label of ['A) Not coded, causal, fix + link', 'A) "Coded, causal, fix + link"', 'A) Reference']) {
|
||||
const t = retryTranscript(), c = t.calls[6]!, q = c.questions[0]!; q.options[0]!.label = label; c.answers = { [q.question]: label }; expect(devexSeedCoverage(t).complete).toBe(false);
|
||||
}
|
||||
for (const prefix of ['Source. ', 'If approved, ']) {
|
||||
const t = retryTranscript(), c = t.calls[6]!, q = c.questions[0]!; q.options[0]!.description = prefix + q.options[0]!.description; expect(devexSeedCoverage(t).complete).toBe(false);
|
||||
}
|
||||
const pending = retryTranscript(); pending.calls[5]!.answered = false; expect(devexSeedCoverage(pending).complete).toBe(false);
|
||||
const foreign = retryTranscript(); foreign.calls[6]!.sessionId = 'foreign'; expect(devexSeedCoverage(foreign).complete).toBe(false);
|
||||
});
|
||||
test('the focused source and exact fixture select only DX; dependency arrays stay dense', () => {
|
||||
for (const file of ['test/dx-asserted-defect-as.test.ts', 'test/fixtures/dx-asserted-defect-as.json', 'test/fixtures/dx-asserted-defect-as-retry.json']) {
|
||||
expect(Object.entries(E2E_TOUCHFILES).filter(([, patterns]) => patterns.some(p => matchGlob(file, p))).map(([name]) => name)).toEqual(['plan-devex-finding-count']);
|
||||
}
|
||||
for (const files of Object.values(E2E_TOUCHFILES)) for (let i = 0; i < files.length; i++) { expect(Object.hasOwn(files, i)).toBe(true); expect(typeof files[i]).toBe('string'); }
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,115 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
import captured from './fixtures/eng-cache-writes-as.json';
|
||||
import { engFirstReviewAUQ, engSetupAUQ, engStep0Boundary, nativePlanCallFingerprint, planCountQuestionPhase } from './helpers/claude-pty-runner';
|
||||
import type { NativePlanQuestionCall } from './helpers/plan-count-transcript';
|
||||
import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles';
|
||||
|
||||
const actual = () => structuredClone(captured.call) as NativePlanQuestionCall;
|
||||
function answered(c: NativePlanQuestionCall, index = 0) {
|
||||
c.answers = { [c.questions[0]!.question]: c.questions[0]!.options[index]!.label };
|
||||
return nativePlanCallFingerprint(c, 0, true);
|
||||
}
|
||||
function allText(edit: (s: string) => string) {
|
||||
const c = actual(), q = c.questions[0]!; q.question = edit(q.question);
|
||||
for (const o of q.options) { o.label = edit(o.label); o.description = edit(o.description ?? ''); }
|
||||
return c;
|
||||
}
|
||||
|
||||
test('the exact completed retry starts review with the current cache ownership decision', () => {
|
||||
const c = actual(), before = JSON.stringify(c), fp = nativePlanCallFingerprint(c, 0, true);
|
||||
expect(engFirstReviewAUQ(fp)).toBe(true); expect(engSetupAUQ(fp)).toBe(false);
|
||||
expect(planCountQuestionPhase(fp, false, engStep0Boundary, engFirstReviewAUQ, engSetupAUQ)).toMatchObject({ preReview: false, reviewStarted: true });
|
||||
expect(JSON.stringify(c)).toBe(before); expect(captured.provenance.retrospectivePass).toBe(false);
|
||||
});
|
||||
|
||||
test('all offered choices and consistently renamed actors retain the same review identity', () => {
|
||||
for (const reverse of [false, true]) for (let i = 0; i < 3; i++) {
|
||||
const c = actual(); if (reverse) c.questions[0]!.options.reverse();
|
||||
expect(engFirstReviewAUQ(answered(c, i))).toBe(true);
|
||||
}
|
||||
for (const [one, two] of [['One', 'Two'], ['$Reader', '_Writer'], ['SessionMint', 'AuthBroker']]) {
|
||||
const c = allText(t => t.replaceAll('AuthBroker', '__one__').replaceAll('SessionMint', two).replaceAll('__one__', one));
|
||||
expect(engFirstReviewAUQ(answered(c))).toBe(true);
|
||||
}
|
||||
const c = allText(t => t.replace(/^D2 /, 'D17 ').replace(/\b2([A-C])\b/g, '17$1'));
|
||||
expect(engFirstReviewAUQ(answered(c))).toBe(true);
|
||||
});
|
||||
|
||||
test('native completion, session, exact answer and option binding remain mandatory', () => {
|
||||
const mutations: Array<(c: NativePlanQuestionCall) => void> = [
|
||||
c => { c.answered = false; }, c => { c.failed = true; }, c => { c.answers = {}; },
|
||||
c => { c.answers = { [c.questions[0]!.question]: 'unoffered' }; }, c => { c.answers!['foreign'] = 'answer'; },
|
||||
c => { c.answeredAt = 'invalid'; }, c => { c.unansweredQuestionIndices = [0]; },
|
||||
c => { c.sessionId = ''; }, c => { c.toolUseId = ''; }, c => { c.questions[0]!.multiSelect = true; },
|
||||
c => { c.questions.push(structuredClone(c.questions[0]!)); }, c => { c.questions[0]!.options[1]!.label = c.questions[0]!.options[0]!.label; },
|
||||
];
|
||||
for (const mutate of mutations) { const c = actual(); mutate(c); expect(engFirstReviewAUQ(nativePlanCallFingerprint(c, 0, true))).toBe(false); }
|
||||
const fp = answered(actual());
|
||||
expect(engFirstReviewAUQ({ ...fp, signature: 'foreign' })).toBe(false);
|
||||
expect(engFirstReviewAUQ({ ...fp, nativeQuestionIndex: 1 })).toBe(false);
|
||||
expect(engFirstReviewAUQ({ ...fp, options: [...fp.options].reverse() })).toBe(false);
|
||||
});
|
||||
|
||||
test('title, own context, current assessment and two distinct writers are required', () => {
|
||||
for (const edit of [
|
||||
(t: string) => 'Source.\n' + t, (t: string) => '> ' + t, (t: string) => '```\n' + t + '\n```',
|
||||
(t: string) => t.replace('Who is allowed', 'Who was allowed'),
|
||||
(t: string) => t.replace('the auth cache?', 'the billing cache?'),
|
||||
(t: string) => t.replace('Project/branch/task:', 'Earlier review:'),
|
||||
(t: string) => t.replace('AuthBroker and SessionMint both', 'AuthBroker and AuthBroker both'),
|
||||
(t: string) => t.replace('both mutating one backing cache', 'both previously mutating one backing cache'),
|
||||
(t: string) => t.replace('ELI10: Two services', 'ELI10: Source. Two services'),
|
||||
(t: string) => t.replace('ELI10: Two services', 'ELI10: If approved, two services'),
|
||||
(t: string) => t.replace('nothing orders their writes.', 'their writes are serialized.'),
|
||||
(t: string) => t.replace('Project/branch/task: ', 'Project/branch/task: Assuming approval, '),
|
||||
(t: string) => t.replace('Project/branch/task: ', 'Project/branch/task: Source. '),
|
||||
(t: string) => t.replace('Multi-tenant Auth Refactor,', 'Multi-tenant Auth Refactor if approved,'),
|
||||
]) { const c = actual(); c.questions[0]!.question = edit(c.questions[0]!.question); expect(engFirstReviewAUQ(answered(c))).toBe(false); }
|
||||
for (const header of ['Scope', 'Issue 1', 'Report', 'Cache examples']) { const c = actual(); c.questions[0]!.header = header; expect(engFirstReviewAUQ(answered(c))).toBe(false); }
|
||||
});
|
||||
|
||||
test('owned current status beats a matching assertion while archived and foreign status does not', () => {
|
||||
for (const status of ['withdrawn', 'superseded', 'rejected', 'cancelled', 'closed', 'hypothetical', 'not current', 'no longer current']) {
|
||||
for (const [open, close] of [['', ''], ['"', '"'], ["'", "'"], ['“', '”'], ['‘', '’'], ['`', '`']]) {
|
||||
for (const target of [-1, 0, 2]) for (const owner of ['This finding', 'D2']) {
|
||||
const c = actual(), q = c.questions[0]!, suffix = `\nCorrection: ${owner} is ${open}${status}${close}.`;
|
||||
if (target < 0) q.question += suffix; else q.options[target]!.description += suffix;
|
||||
expect(engFirstReviewAUQ(answered(c)), `${target}: ${owner} ${open}${status}${close}`).toBe(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const tail of ['D29 is withdrawn.', '> This finding is withdrawn.', 'The prior report said "This finding is withdrawn."', 'An archived review recorded this finding is "withdrawn".', 'An archived review recorded this finding is \'withdrawn\'.', '```\nThis finding is withdrawn.\n```']) {
|
||||
for (const target of [-1, 0, 2]) { const c = actual(), q = c.questions[0]!;
|
||||
if (target < 0) q.question += '\n' + tail; else q.options[target]!.description += '\n' + tail;
|
||||
expect(engFirstReviewAUQ(answered(c)), `${target}: ${tail}`).toBe(true);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('a remedy and opposed choice must bind the same current writers and active race', () => {
|
||||
for (const index of [0, 2]) for (const prefix of ['Source. ', 'If approved, ', 'Assuming approval, ', 'Historical assessment: ', '> ', '"']) {
|
||||
const c = actual(), o = c.questions[0]!.options[index]!; o.description = prefix + o.description + (prefix === '"' ? '"' : '');
|
||||
expect(engFirstReviewAUQ(answered(c))).toBe(false);
|
||||
}
|
||||
for (const index of [0, 1]) for (const name of ['Foreign', 'AuthBroker']) {
|
||||
const c = actual(), o = c.questions[0]!.options[index]!; o.description = o.description!.replace('SessionMint', name);
|
||||
expect(engFirstReviewAUQ(answered(c))).toBe(false);
|
||||
}
|
||||
for (const [target, tail] of [
|
||||
[-1, 'The services no longer mutate the cache.'], [-1, 'Correction: AuthBroker no longer writes to the cache.'],
|
||||
[-1, 'The writes are now serialized.'], [-1, 'The writers are now serialized.'], [2, 'Correction: The writers are now serialized.'], [0, 'Correction: AuthBroker also writes to the cache.'],
|
||||
[0, 'The adapter accepts stale writes.'], [0, 'The version check is optional.'],
|
||||
[2, 'The race is resolved.'], [2, 'Correction: Do not keep both writers.'],
|
||||
[2, 'Only SessionMint writes to the cache.'], [2, 'Both writers no longer mutate the cache.'],
|
||||
] as const) {
|
||||
const c = actual(), q = c.questions[0]!; if (target < 0) q.question += '\n' + tail; else q.options[target]!.description += '\n' + tail;
|
||||
expect(engFirstReviewAUQ(answered(c)), `${target}: ${tail}`).toBe(false);
|
||||
}
|
||||
for (const i of [0, 2]) { const c = actual(); c.questions[0]!.options[i]!.label = `2${i ? 'C' : 'A'} Record the report`; expect(engFirstReviewAUQ(answered(c))).toBe(false); }
|
||||
});
|
||||
|
||||
test('new source and exact public fixture select both Eng boundary owners', () => {
|
||||
for (const file of ['test/helpers/eng-cache-writer-decision.ts', 'test/eng-cache-writes-as.test.ts', 'test/fixtures/eng-cache-writes-as.json']) {
|
||||
expect(selectTests([file], E2E_TOUCHFILES, []).selected.sort()).toEqual(['plan-eng-finding-count', 'plan-eng-multi-finding-batching']);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,126 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
import captured from './fixtures/eng-declarative-as.json';
|
||||
import { engFirstReviewAUQ, engSetupAUQ, engStep0Boundary, nativePlanCallFingerprint, planCountQuestionPhase } from './helpers/claude-pty-runner';
|
||||
import type { NativePlanQuestionCall } from './helpers/plan-count-transcript';
|
||||
import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles';
|
||||
|
||||
const actual = () => structuredClone(captured.call) as NativePlanQuestionCall;
|
||||
function answered(c: NativePlanQuestionCall, index = 0) {
|
||||
c.answers = { [c.questions[0]!.question]: c.questions[0]!.options[index]!.label };
|
||||
return nativePlanCallFingerprint(c, 0, true);
|
||||
}
|
||||
function edit(replace: (text: string) => string) {
|
||||
const c = actual(), q = c.questions[0]!;
|
||||
q.question = replace(q.question);
|
||||
q.options.forEach(o => { o.label = replace(o.label); o.description = replace(o.description ?? ''); });
|
||||
return c;
|
||||
}
|
||||
|
||||
test('a completed declarative cache issue starts review without a question mark or qid', () => {
|
||||
const fp = answered(actual());
|
||||
expect(engFirstReviewAUQ(fp)).toBe(true);
|
||||
expect(engSetupAUQ(fp)).toBe(false);
|
||||
expect(planCountQuestionPhase(fp, false, engStep0Boundary, engFirstReviewAUQ, engSetupAUQ)).toMatchObject({ preReview: false, reviewStarted: true });
|
||||
expect(captured.provenance.retrospectivePass).toBe(false);
|
||||
});
|
||||
|
||||
test('all answers, option orders, identifiers and consistent ordinals qualify', () => {
|
||||
for (const reverse of [false, true]) for (let i = 0; i < 3; i++) {
|
||||
const c = actual(); if (reverse) c.questions[0]!.options.reverse();
|
||||
expect(engFirstReviewAUQ(answered(c, i))).toBe(true);
|
||||
}
|
||||
for (const name of ['TenantCache', '$Shared', '_Store']) expect(engFirstReviewAUQ(answered(edit(t => t.replaceAll('AuthCache', name))))).toBe(true);
|
||||
for (const [one, two] of [['First', 'Second'], ['Z_store', '$Reader'], ['SessionMint', 'AuthBroker']]) {
|
||||
expect(engFirstReviewAUQ(answered(edit(t => t.replaceAll('AuthBroker', '__first__').replaceAll('SessionMint', two).replaceAll('__first__', one))))).toBe(true);
|
||||
}
|
||||
for (const kind of ['Issue', 'Finding']) expect(engFirstReviewAUQ(answered(edit(t => t.replace('Issue 1 ', `${kind} 17 `).replace(/\b1([A-C])\b/g, '17$1'))))).toBe(true);
|
||||
});
|
||||
|
||||
test('native completion and matching answered menu remain required', () => {
|
||||
const mutations: Array<(c: NativePlanQuestionCall) => void> = [
|
||||
c => { c.answered = false; }, c => { c.failed = true; }, c => { c.answers = {}; },
|
||||
c => { c.answers = { [c.questions[0]!.question]: 'unoffered' }; }, c => { c.answers!['foreign'] = 'answer'; },
|
||||
c => { c.answeredAt = 'invalid'; }, c => { c.unansweredQuestionIndices = [0]; },
|
||||
c => { c.sessionId = ''; }, c => { c.toolUseId = ''; }, c => { c.questions[0]!.multiSelect = true; },
|
||||
c => { c.questions.push(structuredClone(c.questions[0]!)); },
|
||||
c => { c.questions[0]!.options[1]!.label = c.questions[0]!.options[0]!.label; },
|
||||
];
|
||||
for (const mutate of mutations) { const c = actual(); mutate(c); expect(engFirstReviewAUQ(nativePlanCallFingerprint(c, 0, true))).toBe(false); }
|
||||
const fp = answered(actual());
|
||||
expect(engFirstReviewAUQ({ ...fp, signature: 'foreign' })).toBe(false);
|
||||
expect(engFirstReviewAUQ({ ...fp, nativeQuestionIndex: 1 })).toBe(false);
|
||||
expect(engFirstReviewAUQ({ ...fp, options: [...fp.options].reverse() })).toBe(false);
|
||||
});
|
||||
|
||||
test('the brief must own a current architecture assessment and distinct writers', () => {
|
||||
const changes = [
|
||||
(t: string) => 'Example: ' + t, (t: string) => '> ' + t, (t: string) => '```\n' + t + '\n```',
|
||||
(t: string) => t.replace('Section 1 Architecture', 'Section 1 Administration'),
|
||||
(t: string) => t.replace('ELI10: Two services', 'ELI10: If two services'),
|
||||
(t: string) => t.replace('ELI10: Two services', 'ELI10: Source example: Two services'),
|
||||
(t: string) => t.replace('AuthBroker and SessionMint share', 'AuthBroker and AuthBroker share'),
|
||||
(t: string) => t.replace('share a global mutable', 'used to share a global mutable'),
|
||||
(t: string) => t.replace('writes can interleave', 'writes are serialized').replace('no ordering', 'per-key ordering'),
|
||||
(t: string) => t.replace('Project/branch/task:', 'Historical assessment:'),
|
||||
];
|
||||
for (const change of changes) { const c = actual(); c.questions[0]!.question = change(c.questions[0]!.question); expect(engFirstReviewAUQ(answered(c))).toBe(false); }
|
||||
for (const header of ['Setup', 'TODOs', 'Issue 2', 'Review report']) { const c = actual(); c.questions[0]!.header = header; expect(engFirstReviewAUQ(answered(c))).toBe(false); }
|
||||
});
|
||||
|
||||
test('same-decision withdrawals and contrary current state invalidate the issue', () => {
|
||||
for (const status of ['withdrawn', 'superseded', 'rejected', 'cancelled', 'resolved', 'closed', 'not current', 'no longer current']) {
|
||||
for (const literal of [status, `"${status}"`, `“${status}”`, `'${status}'`, `‘${status}’`, '`' + status + '`']) for (const target of ['question', 'remedy', 'unchanged']) {
|
||||
const c = actual(), q = c.questions[0]!, suffix = ` This finding is ${literal}.`;
|
||||
if (target === 'question') q.question += suffix; else q.options[target === 'remedy' ? 0 : 2]!.description += suffix;
|
||||
expect(engFirstReviewAUQ(answered(c))).toBe(false);
|
||||
}
|
||||
}
|
||||
for (const contradiction of ['The cache is no longer global.', 'The services no longer mutate shared state.', 'No current risk remains.']) {
|
||||
const c = actual(); c.questions[0]!.question += '\n' + contradiction; expect(engFirstReviewAUQ(answered(c))).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('technical options cannot be quoted, hypothetical, mismatched or cancelled', () => {
|
||||
for (const index of [0, 2]) for (const frame of ['Example: ', 'If approved: ', 'Source excerpt: ', '> ']) {
|
||||
const c = actual(); c.questions[0]!.options[index]!.description = frame + c.questions[0]!.options[index]!.description;
|
||||
expect(engFirstReviewAUQ(answered(c))).toBe(false);
|
||||
}
|
||||
for (const [index, suffix] of [[0, ' Correction: Do not remove the module-level export.'], [0, ' The module export remains.'], [0, ' Writes remain unordered.'], [2, ' Correction: Do not proceed as written.'], [2, ' The race is resolved.']] as const) {
|
||||
const c = actual(); c.questions[0]!.options[index]!.description += suffix; expect(engFirstReviewAUQ(answered(c))).toBe(false);
|
||||
}
|
||||
for (const index of [0, 2]) { const c = actual(); c.questions[0]!.options[index]!.label = '1' + (index ? 'C' : 'A') + ') Record in report'; expect(engFirstReviewAUQ(answered(c))).toBe(false); }
|
||||
const c = actual(); c.questions[0]!.options[0]!.description = c.questions[0]!.options[0]!.description!.replaceAll('AuthCache', 'UnrelatedCache');
|
||||
expect(engFirstReviewAUQ(answered(c))).toBe(false);
|
||||
});
|
||||
|
||||
test('new boundary regressions select the two Eng count owners', () => {
|
||||
for (const file of ['test/eng-declarative-as.test.ts', 'test/fixtures/eng-declarative-as.json']) expect(selectTests([file], E2E_TOUCHFILES, []).selected.sort()).toEqual(['plan-eng-finding-count', 'plan-eng-multi-finding-batching']);
|
||||
});
|
||||
|
||||
|
||||
test('current named-owner contradictions are distinct from foreign and archived references', () => {
|
||||
for (const statement of ['AuthCache is no longer global.', 'AuthCache is no longer mutable.', 'AuthBroker no longer mutates the cache.', 'SessionMint no longer writes to the cache.', 'D4 is withdrawn.', 'Correction: This finding is withdrawn.', 'Correction: Issue 1 is "withdrawn".', 'Correction: AuthCache is no longer global.', "Issue 1 is 'withdrawn'.", 'This finding is “withdrawn”.']) {
|
||||
for (const boundary of ['\n', '; ']) {
|
||||
const c = actual(); c.questions[0]!.question += boundary + statement;
|
||||
expect(engFirstReviewAUQ(answered(c))).toBe(false);
|
||||
}
|
||||
}
|
||||
for (const statement of ['Issue 19 is withdrawn.', 'D42 is withdrawn.', 'AnotherCache is no longer global.', 'An archived review recorded this finding is "withdrawn".', "An archived review recorded this finding is 'withdrawn'.", 'The prior report said "This finding is withdrawn."', '> This finding is withdrawn.']) {
|
||||
const c = actual(); c.questions[0]!.question += '\n' + statement;
|
||||
expect(engFirstReviewAUQ(answered(c))).toBe(true);
|
||||
}
|
||||
for (const replacement of ['an unrelated billing cache', 'a different cache', 'an OtherCache']) {
|
||||
const c = actual(); c.questions[0]!.options[2]!.description = c.questions[0]!.options[2]!.description!.replace('an auth cache', replacement);
|
||||
expect(engFirstReviewAUQ(answered(c))).toBe(false);
|
||||
}
|
||||
const c = actual(); c.questions[0]!.options[2]!.description = c.questions[0]!.options[2]!.description!.replace('an auth cache', 'an AuthCache');
|
||||
expect(engFirstReviewAUQ(answered(c))).toBe(true);
|
||||
});
|
||||
|
||||
|
||||
test('the unchanged option cannot contradict its own remaining cache risk', () => {
|
||||
for (const statement of ['Correction: The writers are now serialized.', 'AuthCache is no longer global.', 'The cache is removed.']) {
|
||||
const c = actual(); c.questions[0]!.options[2]!.description += '\n' + statement;
|
||||
expect(engFirstReviewAUQ(answered(c))).toBe(false);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,93 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { evaluateEngSeedCoverage } from './helpers/eng-seeded-coverage';
|
||||
import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles';
|
||||
|
||||
// Exact public Write acknowledged in the first AS attempt. The paid failure
|
||||
// stays failed; this fixture verifies only the report's mandatory baseline.
|
||||
const report = readFileSync(new URL('./fixtures/eng-mandatory-baseline-as.md', import.meta.url), 'utf8');
|
||||
const declaration = report.match(/^### CRITICAL — regression \(mandatory, REGRESSION RULE\)\n[\s\S]*?(?=\n### )/m)![0];
|
||||
const task = report.match(/^- \[ \] \*\*T5 .*\n(?: .*(?:\n|$))*/m)![0];
|
||||
const compact = '# Current reviewed plan\n\n## Tests\n\n' + declaration + '\n## Implementation Tasks\n' + task;
|
||||
const check = (text: string) => evaluateEngSeedCoverage({ status: 'ready', calls: [], assistantMessages: [] }, text, 0, 1);
|
||||
|
||||
const negative: Array<[string, (s: string) => string]> = [
|
||||
['missing declaration', s => s.replace(declaration, '')],
|
||||
['optional heading', s => s.replace('mandatory, REGRESSION RULE', 'optional, REGRESSION RULE')],
|
||||
['historical owner', s => s.replace('## Tests', '## Historical tests')],
|
||||
['source ancestor', s => '# Source excerpt\n' + s.replace('# Current reviewed plan\n', '')],
|
||||
['bare source introduction', s => 'Source:\n\n' + s.replace('# Current reviewed plan\n', '')],
|
||||
['conditional declaration', s => s.replace('`legacyAuthFlow()` is', 'If approved, `legacyAuthFlow()` is')],
|
||||
['source declaration', s => s.replace('`legacyAuthFlow()` is', 'Source:\n`legacyAuthFlow()` is')],
|
||||
['quoted declaration', s => s.replace(declaration, declaration.split('\n').map(l => '> ' + l).join('\n'))],
|
||||
['fenced declaration', s => s.replace(declaration, '```\n' + declaration + '\n```')],
|
||||
['literal declaration', s => s.replace(declaration, declaration.replace(/`/g, '').split('\n').map(l => '`' + l + '`').join('\n'))],
|
||||
['quoted declaration sentence', s => s.replace('Before any rewrite:', '"Before any rewrite:').replace('inputs. This suite', 'inputs." This suite')],
|
||||
['wrong legacy target', s => s.replaceAll('legacyAuthFlow', 'otherAuthFlow')],
|
||||
['capture after rewrite', s => s.replace('Before any rewrite:', 'After the rewrite:')],
|
||||
['proposed outputs', s => s.replace('records current', 'records proposed')],
|
||||
['new path only', s => s.replace('against the legacy path now', 'against the new path now')],
|
||||
['optional assertion', s => s.replace('records current', 'may record current')],
|
||||
['missing baseline task', s => s.replace(task, '')],
|
||||
['historical task owner', s => s.replace('## Implementation Tasks', '## Historical Implementation Tasks')],
|
||||
['conditional task', s => s.replace(task, 'If approved:\n' + task)],
|
||||
['source task', s => s.replace(task, 'Source:\n' + task)],
|
||||
['quoted task', s => s.replace(task, task.split('\n').map(l => '> ' + l).join('\n'))],
|
||||
['wrong file', s => s.replace(' - Files: tests/auth/legacyAuthFlow.characterization.test.ts', ' - Files: tests/auth/other.test.ts')],
|
||||
['missing same-file binding', s => s.replace(' - Files: tests/auth/legacyAuthFlow.characterization.test.ts\n', '')],
|
||||
['wrong task subject', s => s.replace('suite for `legacyAuthFlow()` current behavior', 'suite for `otherAuthFlow()` current behavior')],
|
||||
['missing verification', s => s.replace(' - Verify: suite green against unmodified legacy before any other task merges', '')],
|
||||
['changed baseline', s => s.replace('against unmodified legacy', 'against modified legacy')],
|
||||
['baseline after merge', s => s.replace('before any other task merges', 'after every other task merges')],
|
||||
['missing before-merge gate', s => s.replace(' before any other task merges', '')],
|
||||
['neighboring verification', s => s.replace(' - Verify:', '- [ ] T6 — tests/auth — Another suite\n - Verify:')],
|
||||
['duplicate task identity', s => s.replace(task, task + task)],
|
||||
...['Source:', 'If approved:', 'Assuming approval,', 'Provided approval,', 'Once approved:', 'When approved:', 'Pending approval:'].map(prefix =>
|
||||
[`verification owner ${prefix}`, (s: string) => s.replace(' - Verify:', ` ${prefix}\n - Verify:`)] as [string, (s: string) => string]),
|
||||
...['withdrawn', 'superseded', 'optional', 'not current', 'no longer current', 'no longer required'].flatMap(status => [
|
||||
[`current T5 ${status}`, (s: string) => s + `\n## Current assessment\nT5 is ${status}.\n`],
|
||||
[`quoted T5 ${status}`, (s: string) => s + `\n## Current assessment\nT5 is "${status}".\n`],
|
||||
[`baseline ${status}`, (s: string) => s.replace(task, task + ` This baseline verification is "${status}".\n`)],
|
||||
] as Array<[string, (s: string) => string]>),
|
||||
['current status row', s => s + '\n## Current assessment\n| T5 | Withdrawn |\n'],
|
||||
['quoted status row', s => s + '\n## Current assessment\n| T5 | "Withdrawn" |\n'],
|
||||
['withdrawn legacy suite', s => s + '\n## Current assessment\nThe legacy characterization suite is "withdrawn".\n'],
|
||||
['declaration withdrawn', s => s.replace(declaration, declaration + '\nThis suite is withdrawn.\n')],
|
||||
['baseline changed before task', s => s + '\n## Current assessment\nlegacyAuthFlow() is modified before T5.\n'],
|
||||
];
|
||||
|
||||
describe('mandatory legacy baseline before any other task merges', () => {
|
||||
test('the exact acknowledged report requires the baseline without inventing native decisions', () => {
|
||||
expect(createHash('sha256').update(report).digest('hex')).toBe('60620ddd798a567423087731775557848c260a82080e9eceb1367a9bb9fc5d23');
|
||||
expect(check(report)).toMatchObject({ regression: 'plan', ok: false,
|
||||
missing: ['complexity', 'shared-cache', 'swallowed-errors', 'sequential-idp'] });
|
||||
expect(check(compact).regression).toBe('plan');
|
||||
});
|
||||
test('task numbering, test paths, markup and line wrapping do not change the obligation', () => {
|
||||
for (const altered of [compact.replaceAll('T5', 'T31'), compact.replaceAll('tests/auth', 'test/login'),
|
||||
compact.replaceAll('legacyAuthFlow.characterization.test.ts', 'prior-behavior.test.js'),
|
||||
compact.replace(/[`*]/g, ''), compact.replace(/\n(?=[a-z])/g, ' ')])
|
||||
expect(check(altered).regression).toBe('plan');
|
||||
});
|
||||
test('this baseline does not require an invented same-suite flag-on rerun', () => {
|
||||
const baselineOnly = compact.replace(/ and moves to\n`AuthBroker` when the flag is removed \(TODO 1\)/, '');
|
||||
expect(baselineOnly).not.toBe(compact);
|
||||
expect(check(baselineOnly).regression).toBe('plan');
|
||||
});
|
||||
test('historical quotations, unrelated suite statuses and future completion do not withdraw the baseline', () => {
|
||||
for (const addition of ['\n## History\n"T5 is withdrawn."', '\n## History\n> T5 is withdrawn.',
|
||||
'\n## Historical task status\n| T5 | Withdrawn |', '\n## Current assessment\n| T9 | Withdrawn |',
|
||||
'\n## Payment regression suite\nThe regression suite is withdrawn.',
|
||||
'\n## Current assessment\nIf T5 is withdrawn, reopen the rollout decision.'])
|
||||
expect(check(compact + addition).regression).toBe('plan');
|
||||
expect(check('Source:\n\n' + compact).regression).toBe('plan');
|
||||
});
|
||||
test.each(negative)('%s supplies no mandatory baseline', (_, change) => {
|
||||
const altered = change(compact); expect(altered).not.toBe(compact); expect(check(altered).regression).toBeUndefined();
|
||||
});
|
||||
test('new regression artifacts select only the existing Eng owner', () => {
|
||||
for (const file of ['test/eng-mandatory-baseline-as.test.ts', 'test/fixtures/eng-mandatory-baseline-as.md'])
|
||||
expect(selectTests([file], E2E_TOUCHFILES, []).selected).toEqual(['plan-eng-finding-count']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,130 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { createHash } from 'node:crypto';
|
||||
import fixture from './fixtures/eng-retry-coverage-as.json';
|
||||
import { evaluateEngSeedCoverage } from './helpers/eng-seeded-coverage';
|
||||
import type { NativePlanQuestionCall, PlanCountTranscript } from './helpers/plan-count-transcript';
|
||||
import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles';
|
||||
|
||||
const report = readFileSync(new URL('./fixtures/eng-retry-baseline-as.md', import.meta.url), 'utf8');
|
||||
const transcript = (): PlanCountTranscript => ({ status: 'ready', calls: structuredClone(fixture.calls) as NativePlanQuestionCall[], assistantMessages: [] });
|
||||
const check = (t = transcript(), plan = report) => evaluateEngSeedCoverage(t, plan, 0, Date.parse('2026-09-11T00:00:00Z'));
|
||||
const targets = [[0, 'complexity'], [1, 'shared-cache'], [3, 'swallowed-errors']] as const;
|
||||
function change(t: PlanCountTranscript, i: number, fn: (q: NativePlanQuestionCall['questions'][number]) => void) {
|
||||
const c = t.calls[i]!, q = c.questions[0]!, selected = q.options.findIndex(o => o.label === c.answers![q.question]);
|
||||
fn(q); c.answers = { [q.question]: q.options[selected]!.label };
|
||||
}
|
||||
const declaration = report.match(/^## Tests \(revised[^\n]+\n[\s\S]*?(?=\nCoverage target:)/m)![0];
|
||||
const strategy = report.match(/^## Worktree parallelization strategy\n[\s\S]*?(?=\n## Implementation Tasks)/m)![0];
|
||||
const task = report.match(/^- \[ \] \*\*T1 .*\n(?: .*(?:\n|$))*/m)![0];
|
||||
const compact = '# Current reviewed plan\n\n' + declaration + '\n' + strategy + '\n## Implementation Tasks\n' + task;
|
||||
const baseline = (plan: string) => check({ status: 'ready', calls: [], assistantMessages: [] }, plan).regression;
|
||||
|
||||
describe('Eng retry decisions with owned explanations and concrete repairs', () => {
|
||||
test('the exact nine completed native calls and report supply all required coverage', () => {
|
||||
const t = transcript(), unchanged = JSON.stringify(t), result = check(t);
|
||||
expect(t.calls).toHaveLength(9); expect(result.ok).toBe(true); expect(result.missing).toEqual([]); expect(result.regression).toBe('plan');
|
||||
for (const [i, seed] of targets) expect(result.decisions[seed]).toBe(`${t.calls[i]!.sessionId}:${t.calls[i]!.toolUseId}`);
|
||||
expect(JSON.stringify(t)).toBe(unchanged); expect(fixture.provenance.paidOutcomeReclassified).toBe(false);
|
||||
expect(createHash('sha256').update(report).digest('hex')).toBe('2e4c618d441c463f866aa4bcb10706d59c14236352c1c5c07afc4a0e01874a0d');
|
||||
});
|
||||
test('separate completed decisions count for every offered answer and equivalent ordinals', () => {
|
||||
for (const [i, seed] of targets) for (let answer = 0; answer < 3; answer++) {
|
||||
const t = transcript(), c = t.calls[i]!, q = c.questions[0]!; c.answers = { [q.question]: q.options[answer]!.label };
|
||||
expect(check(t).decisions[seed]).toBeDefined();
|
||||
}
|
||||
for (const [i, seed] of targets) {
|
||||
const t = transcript(); change(t, i, q => { q.question = q.question.replace(/^D\d+/, 'D31'); });
|
||||
expect(check(t).decisions[seed]).toBeDefined();
|
||||
t.calls.splice(i, 1); expect(check(t).decisions[seed]).toBeUndefined();
|
||||
}
|
||||
});
|
||||
test('the current subject and its own explanation must assert the gap', () => {
|
||||
for (const [i, seed] of targets) for (const transform of [
|
||||
(s: string) => 'Source:\n' + s, (s: string) => '> ' + s, (s: string) => '```\n' + s + '\n```',
|
||||
(s: string) => s.replace('ELI10: ', 'ELI10: Source excerpt: '),
|
||||
(s: string) => s.replace('ELI10: ', 'ELI10: If approved, '),
|
||||
(s: string) => s.replace('Project/branch/task: ', 'Project/branch/task: Historical assessment: '),
|
||||
(s: string) => s.replace(/^ELI10:.*$/m, 'ELI10: This behavior already works; there is no current defect.'),
|
||||
(s: string) => s.replace(/\nELI10:/, '\nSource:\nELI10:'),
|
||||
]) { const t = transcript(); change(t, i, q => { q.question = transform(q.question); }); expect(check(t).decisions[seed]).toBeUndefined(); }
|
||||
const t = transcript(); change(t, 3, q => { q.question = q.question.replace('each catch swallows a different error class', 'each catch propagates its error class'); });
|
||||
expect(check(t).decisions['swallowed-errors']).toBeUndefined();
|
||||
});
|
||||
test('same-decision withdrawal wins, while quoted history and another ordinal do not', () => {
|
||||
for (const [i, seed] of targets) for (const status of ['withdrawn', 'superseded', 'hypothetical', 'unproven', 'not current', 'no longer current']) {
|
||||
for (const scalar of [status, `"${status}"`, `'${status}'`, '`' + status + '`']) {
|
||||
const t = transcript(); change(t, i, q => { q.question += `\nCorrection: This finding is ${scalar}.`; }); expect(check(t).decisions[seed]).toBeUndefined();
|
||||
}
|
||||
const t = transcript(); change(t, i, q => { q.question += `\nD${i + 1} is ${status}.`; }); expect(check(t).decisions[seed]).toBeUndefined();
|
||||
}
|
||||
for (const [i, seed] of targets) for (const tail of ['\nD39 is withdrawn.', '\n> This finding is withdrawn.', '\nArchived note: "This finding is withdrawn."', '\nAn archived review recorded this finding is "withdrawn".']) {
|
||||
const t = transcript(); change(t, i, q => { q.question += tail; }); expect(check(t).decisions[seed]).toBeDefined();
|
||||
}
|
||||
});
|
||||
test('repairs belong to current native options, not quoted or narrated recommendations', () => {
|
||||
for (const [i, seed] of targets) for (const mode of ['source', 'conditional', 'quoted', 'withdrawn', 'hypothetical', 'unproven', 'not current', 'no longer current', 'navigation']) {
|
||||
const t = transcript(); change(t, i, q => { q.options.forEach((o, n) => {
|
||||
if (mode === 'source') o.description = 'Source: ' + o.description;
|
||||
else if (mode === 'conditional') o.description = 'If approved, ' + o.description;
|
||||
else if (mode === 'quoted') { o.label = '"' + o.label + '"'; o.description = '"' + o.description + '"'; }
|
||||
else if (mode === 'navigation') { o.label = `Continue ${n}`; o.description = 'Move to the next section.'; }
|
||||
else o.description += `\nThis option is "${mode}".`;
|
||||
}); }); expect(check(t).decisions[seed]).toBeUndefined();
|
||||
}
|
||||
const t = transcript(); change(t, 1, q => { q.options[0]!.description = q.options[0]!.description!.replace('SessionMint writes', 'BillingService writes'); q.options[1]!.description = q.options[1]!.description!.replace('SessionMint writes', 'BillingService writes'); });
|
||||
expect(check(t).decisions['shared-cache']).toBeUndefined();
|
||||
for (const [i, seed] of targets) {
|
||||
const t = transcript(); change(t, i, q => { q.options.forEach(o => { o.description += '\nCorrection: Do not apply this repair.'; }); });
|
||||
expect(check(t).decisions[seed]).toBeUndefined();
|
||||
}
|
||||
});
|
||||
test('native answer, session, time and distinct identity gates are unchanged', () => {
|
||||
for (const [i, seed] of targets) for (const mode of ['unanswered', 'failed', 'pending', 'wrong answer', 'out of time']) {
|
||||
const t = transcript(), c = t.calls[i]!;
|
||||
if (mode === 'unanswered') c.answered = false; else if (mode === 'failed') c.failed = true;
|
||||
else if (mode === 'pending') c.unansweredQuestionIndices = [0]; else if (mode === 'wrong answer') c.answers = { foreign: 'unoffered' }; else c.answeredAt = '2026-09-12T00:00:00Z';
|
||||
expect(check(t).decisions[seed]).toBeUndefined();
|
||||
}
|
||||
for (const mode of ['foreign session', 'duplicate']) { const t = transcript(); if (mode === 'duplicate') t.calls.push(structuredClone(t.calls[0]!)); else t.calls[0]!.sessionId = 'foreign'; expect(check(t).ok).toBe(false); }
|
||||
});
|
||||
});
|
||||
|
||||
describe('Eng retry mandatory baseline is ordered before the changed worktree steps', () => {
|
||||
test('the exact declaration, directory-owned task and merge order bind the legacy baseline', () => {
|
||||
expect(baseline(report)).toBe('plan'); expect(baseline(compact)).toBe('plan');
|
||||
for (const s of [compact.replaceAll('T1', 'T21'), compact.replaceAll('S1', 'S31').replaceAll('S5', 'S35').replaceAll('S6', 'S36'),
|
||||
compact.replaceAll('tests/auth/legacy', 'test/login/prior'), compact.replace(/[`*]/g, '')]) expect(baseline(s)).toBe('plan');
|
||||
});
|
||||
test('a legacy baseline cannot be borrowed from another task, directory, source or later merge', () => {
|
||||
const transforms: Array<(s: string) => string> = [
|
||||
s => s.replace(declaration, ''), s => s.replace(strategy, ''), s => s.replace(task, ''),
|
||||
s => s.replace('mandatory, IRON RULE', 'optional, IRON RULE'), s => s.replace('Before any rewrite, write', 'After the rewrite, write'),
|
||||
s => s.replace('pins current\nbehavior', 'pins proposed\nbehavior'), s => s.replace('legacy path and the new flow', 'new flow only'),
|
||||
s => s.replace('## Tests (revised', '## Historical tests (revised'), s => s.replace('## Implementation Tasks', '## Historical Implementation Tasks'),
|
||||
s => s.replace('## Worktree parallelization strategy', '## Historical worktree parallelization strategy'),
|
||||
s => 'Source:\n' + s.replace('# Current reviewed plan\n', ''), s => s.replace(declaration, declaration.split('\n').map(l => '> ' + l).join('\n')),
|
||||
s => s.replace('Before any rewrite, write', 'If approved, before any rewrite, write'),
|
||||
s => s.replace(task, 'Source:\n' + task), s => s.replace(task, 'If approved:\n' + task),
|
||||
s => s.replace(' - Files: tests/auth/legacy/*', ' - Files: tests/auth/other/*'),
|
||||
s => s.replace(' - Verify:', '- [ ] T2 — tests/auth/other — Another suite\n - Verify:'),
|
||||
s => s.replace('before S5/S6 land', 'after S5/S6 land'), s => s.replace('before S5/S6 land', 'before S2/S4 land'),
|
||||
s => s.replace('; green on both paths after', ''), s => s.replace('Merge A first', 'Merge B first'),
|
||||
s => s.replace('Lane A: S1', 'Lane A: S2'), s => s.replace('| S1 Regression suite', '| S8 Regression suite'),
|
||||
s => s.replace('| S1 Regression suite', '| S1 Other suite | tests/other | — |\n| S1 Regression suite'),
|
||||
s => s.replace('* Lane A: S1 (independent)', '* Lane A: S1 (independent)\n* Lane X: S1 (independent)'),
|
||||
s => s.replace(task, task + task),
|
||||
...['Source:', 'If approved:', 'Once approved:', 'When approved:', 'Pending approval:', 'Assuming approval,'].map(prefix => (s: string) => s.replace(' - Verify:', ` ${prefix}\n - Verify:`)),
|
||||
...['T1', 'S1', 'This baseline verification'].flatMap(owner => ['withdrawn', 'superseded', 'no longer current'].map(status => (s: string) => s + `\n## Current assessment\n${owner} is "${status}".\n`)),
|
||||
s => s + '\n## Current assessment\n| T1 | Withdrawn |\n', s => s + '\n## Current assessment\nlegacyAuthFlow() is modified before T1.\n',
|
||||
];
|
||||
for (const transform of transforms) { const s = transform(compact); expect(s).not.toBe(compact); expect(baseline(s)).toBeUndefined(); }
|
||||
});
|
||||
test('archived and unrelated status does not cancel the current baseline', () => {
|
||||
for (const tail of ['\n## History\n"T1 is withdrawn."', "\n## History\n'T1 is withdrawn.'", '\n## Current assessment\nIf T1 is withdrawn, reopen the rollout decision.', '\n## Current assessment\n| T99 | Withdrawn |', '\n## Historical status\n| T1 | Withdrawn |', '\n## Payment regression suite\nThe regression suite is withdrawn.']) expect(baseline(compact + tail)).toBe('plan');
|
||||
});
|
||||
test('new exact fixtures select only the existing Eng coverage owner', () => {
|
||||
for (const path of ['test/eng-retry-coverage-as.test.ts', 'test/fixtures/eng-retry-coverage-as.json', 'test/fixtures/eng-retry-baseline-as.md'])
|
||||
expect(selectTests([path], E2E_TOUCHFILES, []).selected).toEqual(['plan-eng-finding-count']);
|
||||
});
|
||||
});
|
||||
+1590
File diff suppressed because it is too large.
Load diff
+124
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"provenance": {
|
||||
"run": "ship-source-as-delta-paid-20260910-v1",
|
||||
"sourceHead": "f26d569e0345cb1131d9ca52d4a43965085c3468",
|
||||
"originalAttemptOutcomes": [
|
||||
"failed",
|
||||
"failed"
|
||||
],
|
||||
"paidOutcomesReclassified": false,
|
||||
"projection": "Exact final public assistant text and the native parent file-read pair; init and event envelopes retain only required identity fields."
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"attempt": 1,
|
||||
"cwd": "/tmp/gstack-paid-shard-eMtnp5/tmp/skill-e2e-review-coverage-5yDBnd",
|
||||
"result": {
|
||||
"exitReason": "success",
|
||||
"browseErrors": [],
|
||||
"output": "Test coverage audit is complete. Vitest is not installed in this sandbox, so the suite could not be executed; the analysis below is static, based on reading both files.\n\n```\nTEST COVERAGE MAP \u2014 src/billing.ts (branch: feature/billing vs main)\nLegend: [\u2713] tested [\u2717] GAP \u2500\u2500\u25ba branch\n\nsrc/billing.ts\n\u2502\n\u251c\u2500\u2500 processPayment(amount, currency) 1 of 3 paths tested\n\u2502 \u2502\n\u2502 \u251c\u2500\u2500\u25ba line 3 amount <= 0 \u2192 throw 'Invalid amount' [\u2717] GAP Missing Negative-Path\n\u2502 \u2502 zero / negative boundary never exercised\n\u2502 \u2502\n\u2502 \u251c\u2500\u2500\u25ba line 4 currency \u2209 {USD, EUR} \u2192 throw [\u2717] GAP Missing Negative-Path\n\u2502 \u2502 'Unsupported currency' (e.g. 'GBP', '')\n\u2502 \u2502\n\u2502 \u2514\u2500\u2500\u25ba line 5 return { status: 'success', ... } [\u2713] test/billing.test.ts:6\n\u2502 'processes valid payment' (100, 'USD')\n\u2502 note: only USD covered; EUR happy path untested\n\u2502\n\u2514\u2500\u2500 refundPayment(paymentId, reason) 0 of 3 paths tested\n \u2502 [\u2717] GAP Coverage Gap\n \u2502 exported function, never imported by any test\n \u2502\n \u251c\u2500\u2500\u25ba line 9 !paymentId \u2192 throw 'Payment ID required' [\u2717] GAP Missing Negative-Path\n \u2502\n \u251c\u2500\u2500\u25ba line 10 !reason \u2192 throw 'Reason required' [\u2717] GAP Missing Negative-Path\n \u2502\n \u2514\u2500\u2500\u25ba line 11 return { status: 'refunded', ... } [\u2717] GAP Coverage Gap\n\n\nSUMMARY\n Functions exported ........ 2 covered: 1 uncovered: 1\n Branches / paths .......... 6 covered: 1 uncovered: 5\n Guard clauses ............. 4 covered: 0 uncovered: 4\n Error branches tested ..... 0 of 4\n```\n\nFindings, ranked by severity:\n\n- **refundPayment has zero coverage** (Coverage Gaps). A new public function is exported and never imported in the test file. Both guard clauses and the success return are unverified.\n- **processPayment rejects are untested** (Missing Negative-Path Tests). Both throw branches at lines 3 and 4 have no failing-input test. A regression that removes either guard would pass the suite.\n- **Happy path is narrow.** The one existing test covers only USD. The EUR branch of the currency check is accepted by code but never asserted.\n\nSuggested test stubs to close the gaps:\n\n```ts\nimport { processPayment, refundPayment } from '../src/billing';\n\ntest('rejects zero amount', () => expect(() => processPayment(0, 'USD')).toThrow('Invalid amount'));\ntest('rejects negative amount', () => expect(() => processPayment(-5, 'USD')).toThrow('Invalid amount'));\ntest('rejects unsupported currency', () => expect(() => processPayment(10, 'GBP')).toThrow('Unsupported currency'));\ntest('accepts EUR', () => expect(processPayment(10, 'EUR').currency).toBe('EUR'));\n\ntest('refunds valid payment', () => expect(refundPayment('p1', 'dup').status).toBe('refunded'));\ntest('rejects missing paymentId', () => expect(() => refundPayment('', 'dup')).toThrow('Payment ID required'));\ntest('rejects missing reason', () => expect(() => refundPayment('p1', '')).toThrow('Reason required'));\n```\n\nOne environment note: the package.json test script only echoes \"no tests yet\" and node_modules is absent, so even the existing test does not run in CI as configured.",
|
||||
"transcript": [
|
||||
{
|
||||
"type": "system",
|
||||
"subtype": "init",
|
||||
"cwd": "/tmp/gstack-paid-shard-eMtnp5/tmp/skill-e2e-review-coverage-5yDBnd",
|
||||
"session_id": "7c7ba4bd-9818-4a7e-b3b7-ad74b24c4836"
|
||||
},
|
||||
{
|
||||
"type": "assistant",
|
||||
"session_id": "7c7ba4bd-9818-4a7e-b3b7-ad74b24c4836",
|
||||
"parent_tool_use_id": null,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_015hqy9J6KoorxoLUuXTTEVM",
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "cat -n src/billing.ts && echo ==== && cat -n test/billing.test.ts",
|
||||
"description": "Read source and test files"
|
||||
},
|
||||
"caller": {
|
||||
"type": "direct"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "user",
|
||||
"session_id": "7c7ba4bd-9818-4a7e-b3b7-ad74b24c4836",
|
||||
"parent_tool_use_id": null,
|
||||
"message": {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"tool_use_id": "toolu_015hqy9J6KoorxoLUuXTTEVM",
|
||||
"type": "tool_result",
|
||||
"content": " 1\t\n 2\texport function processPayment(amount: number, currency: string) {\n 3\t if (amount <= 0) throw new Error('Invalid amount');\n 4\t if (currency !== 'USD' && currency !== 'EUR') throw new Error('Unsupported currency');\n 5\t return { status: 'success', amount, currency };\n 6\t}\n 7\t\n 8\texport function refundPayment(paymentId: string, reason: string) {\n 9\t if (!paymentId) throw new Error('Payment ID required');\n 10\t if (!reason) throw new Error('Reason required');\n 11\t return { status: 'refunded', paymentId, reason };\n 12\t}\n====\n 1\t\n 2\timport { describe, test, expect } from 'vitest';\n 3\timport { processPayment } from '../src/billing';\n 4\t\n 5\tdescribe('processPayment', () => {\n 6\t test('processes valid payment', () => {\n 7\t const result = processPayment(100, 'USD');\n 8\t expect(result.status).toBe('success');\n 9\t });\n 10\t // GAP: no test for invalid amount\n 11\t // GAP: no test for unsupported currency\n 12\t // GAP: refundPayment not tested at all\n 13\t});",
|
||||
"is_error": false
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"cwd": "/tmp/gstack-paid-shard-eMtnp5/tmp/skill-e2e-review-coverage-5yDBnd",
|
||||
"result": {
|
||||
"exitReason": "success",
|
||||
"browseErrors": [],
|
||||
"output": "The skill docs don't prescribe a diagram format, so here is the audit in an ASCII coverage map. The branch has one commit shared with main, so the audit covers the full file rather than a diff.\n\n```\nsrc/billing.ts \u2014 test coverage map (test/billing.test.ts)\nLegend: [\u2714] tested [\u2718] GAP (no test)\n\nprocessPayment(amount, currency) 1 of 3 paths covered\n\u2502\n\u251c\u2500 if (amount <= 0) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u25ba throw 'Invalid amount' [\u2718] GAP negative-path\n\u2502 boundary values untested: 0, -1, NaN\n\u2502\n\u251c\u2500 if (currency !== 'USD' && !== 'EUR')\n\u2502 \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u25ba throw 'Unsupported currency' [\u2718] GAP negative-path\n\u2502 untested: 'GBP', '', 'usd' (case)\n\u2502 happy-path 'EUR' branch also untested [\u2718] GAP edge-case\n\u2502\n\u2514\u2500 return { status: 'success', amount, currency } [\u2714] processes valid payment\n (asserts only .status; amount/currency echo unchecked)\n\nrefundPayment(paymentId, reason) 0 of 3 paths covered \u2014 never imported\n\u2502\n\u251c\u2500 if (!paymentId) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u25ba throw 'Payment ID required' [\u2718] GAP negative-path\n\u2502 untested: '', undefined\n\u2502\n\u251c\u2500 if (!reason) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u25ba throw 'Reason required' [\u2718] GAP negative-path\n\u2502 untested: '', undefined\n\u2502\n\u2514\u2500 return { status: 'refunded', paymentId, reason } [\u2714]\u2192[\u2718] GAP coverage: public fn, zero tests\n\nSummary\n Branches: 6 total 1 covered 5 gaps\n Functions: 2 total 1 covered 1 with zero coverage\n```\n\nFindings, ranked:\n\n- **Coverage Gaps: refundPayment has zero tests.** Exported public function, never imported by the test file. Both guards and the success return are unverified.\n- **Missing Negative-Path: amount guard untested.** No test that `processPayment(0, 'USD')` or a negative amount throws. A regression to `amount < 0` would pass the suite.\n- **Missing Negative-Path: currency guard untested.** No test that an unsupported code throws. The `EUR` accept path is also unexercised, so the only value proven to work is `USD`.\n- **Missing Negative-Path: refund guards untested.** Neither the missing payment ID nor the missing reason rejection has a failing-input test.\n- **Weak assertion on the one existing test.** It checks `status` only, so a bug that drops or mutates `amount` or `currency` in the return value would go unnoticed.\n\nThe test file's own comments already acknowledge the first three gaps. Adding five `expect(() => ...).toThrow(...)` cases plus one `EUR` happy path and one `refundPayment` success case would bring every branch to covered.",
|
||||
"transcript": [
|
||||
{
|
||||
"type": "system",
|
||||
"subtype": "init",
|
||||
"cwd": "/tmp/gstack-paid-shard-eMtnp5/tmp/skill-e2e-review-coverage-5yDBnd",
|
||||
"session_id": "b36532b9-369f-4dc3-ba70-197663bdfee6"
|
||||
},
|
||||
{
|
||||
"type": "assistant",
|
||||
"session_id": "b36532b9-369f-4dc3-ba70-197663bdfee6",
|
||||
"parent_tool_use_id": null,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01XBJKThc7uaHKvMwCYY9oxC",
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "cat review/specialists/testing.md; echo ======; cat -n src/billing.ts; echo ======; cat -n test/billing.test.ts",
|
||||
"description": "Read testing specialist, source, and tests"
|
||||
},
|
||||
"caller": {
|
||||
"type": "direct"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "user",
|
||||
"session_id": "b36532b9-369f-4dc3-ba70-197663bdfee6",
|
||||
"parent_tool_use_id": null,
|
||||
"message": {
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"tool_use_id": "toolu_01XBJKThc7uaHKvMwCYY9oxC",
|
||||
"type": "tool_result",
|
||||
"content": "# Testing Specialist Review Checklist\n\nScope: Always-on (every review)\nOutput: JSON objects, one finding per line. Schema:\n{\"severity\":\"CRITICAL|INFORMATIONAL\",\"confidence\":N,\"path\":\"file\",\"line\":N,\"category\":\"testing\",\"summary\":\"...\",\"fix\":\"...\",\"fingerprint\":\"path:line:testing\",\"specialist\":\"testing\"}\nOptional: line, fix, fingerprint, evidence, test_stub.\nIf no findings: output `NO FINDINGS` and nothing else.\n\n---\n\n## Categories\n\n### Missing Negative-Path Tests\n- New code paths that handle errors, rejections, or invalid input with NO corresponding test\n- Guard clauses and early returns that are untested\n- Error branches in try/catch, rescue, or error boundaries with no failure-path test\n- Permission/auth checks that are asserted in code but never tested for the \"denied\" case\n\n### Missing Edge-Case Coverage\n- Boundary values: zero, negative, max-int, empty string, empty array, nil/null/undefined\n- Single-element collections (off-by-one on loops)\n- Unicode and special characters in user-facing inputs\n- Concurrent access patterns with no race-condition test\n\n### Test Isolation Violations\n- Tests sharing mutable state (class variables, global singletons, DB records not cleaned up)\n- Order-dependent tests (pass in sequence, fail when randomized)\n- Tests that depend on system clock, timezone, or locale\n- Tests that make real network calls instead of using stubs/mocks\n\n### Flaky Test Patterns\n- Timing-dependent assertions (sleep, setTimeout, waitFor with tight timeouts)\n- Assertions on ordering of unordered results (hash keys, Set iteration, async resolution order)\n- Tests that depend on external services (APIs, databases) without fallback\n- Randomized test data without seed control\n\n### Security Enforcement Tests Missing\n- Auth/authz checks in controllers with no test for the \"unauthorized\" case\n- Rate limiting logic with no test proving it actually blocks\n- Input sanitization with no test for malicious input\n- CSRF/CORS configuration with no integration test\n\n### Coverage Gaps\n- New public methods/functions with zero test coverage\n- Changed methods where existing tests only cover the old behavior, not the new branch\n- Utility functions called from multiple places but tested only indirectly\n======\n 1\t\n 2\texport function processPayment(amount: number, currency: string) {\n 3\t if (amount <= 0) throw new Error('Invalid amount');\n 4\t if (currency !== 'USD' && currency !== 'EUR') throw new Error('Unsupported currency');\n 5\t return { status: 'success', amount, currency };\n 6\t}\n 7\t\n 8\texport function refundPayment(paymentId: string, reason: string) {\n 9\t if (!paymentId) throw new Error('Payment ID required');\n 10\t if (!reason) throw new Error('Reason required');\n 11\t return { status: 'refunded', paymentId, reason };\n 12\t}\n======\n 1\t\n 2\timport { describe, test, expect } from 'vitest';\n 3\timport { processPayment } from '../src/billing';\n 4\t\n 5\tdescribe('processPayment', () => {\n 6\t test('processes valid payment', () => {\n 7\t const result = processPayment(100, 'USD');\n 8\t expect(result.status).toBe('success');\n 9\t });\n 10\t // GAP: no test for invalid amount\n 11\t // GAP: no test for unsupported currency\n 12\t // GAP: refundPayment not tested at all\n 13\t});",
|
||||
"is_error": false
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+534
@@ -0,0 +1,534 @@
|
||||
{
|
||||
"calls": [
|
||||
{
|
||||
"sessionId": "22820c07-b1f9-4c7c-a7b8-ba7acb607f01",
|
||||
"toolUseId": "toolu_01RLY62gnoEWnQcvsFEe3C9J",
|
||||
"questions": [
|
||||
{
|
||||
"header": "Routing",
|
||||
"question": "D1 — Add gstack skill routing rules to CLAUDE.md?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: gstack works best when your project's CLAUDE.md tells the agent which skill to invoke for which request (bugs → /investigate, ship → /ship). Without it you invoke skills by hand each time. Plan mode forbids editing CLAUDE.md now, so option A would be applied after this review exits plan mode.\nStakes if we pick wrong: mild; you lose auto-routing or gain a section you did not want in a fixture repo.\nRecommendation: A because routing is free once written and removable in one edit.\nNote: options differ in kind, not coverage — no completeness score.\nNet: convenience later vs a tidier CLAUDE.md now.",
|
||||
"options": [
|
||||
{
|
||||
"label": "Add routing rules (recommended)",
|
||||
"description": "✅ Future requests auto-route to the matching skill without typing slash commands. ✅ One committed section, easy to delete. ❌ Adds ~20 lines to a fixture CLAUDE.md; applied only after plan mode ends."
|
||||
},
|
||||
{
|
||||
"label": "No thanks, manual",
|
||||
"description": "✅ CLAUDE.md stays minimal for this fixture repo. ✅ Sets routing_declined so this is never asked again here. ❌ You keep typing skill names by hand."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
},
|
||||
{
|
||||
"header": "Learnings",
|
||||
"question": "D2 — Enable cross-project learnings search?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: gstack stores small lessons per project. With this on, reviews can also read lessons from your other projects on this machine. Nothing leaves your machine. Good for solo developers; skip if you work across client codebases where mixing is a concern.\nStakes if we pick wrong: on for a multi-client machine could surface one client's patterns in another's review; off means slower compounding.\nRecommendation: A because this is a local, single-user sandbox with no client separation concern.\nNote: options differ in kind, not coverage — no completeness score.\nNet: faster compounding vs strict per-project isolation.",
|
||||
"options": [
|
||||
{
|
||||
"label": "Enable cross-project (recommended)",
|
||||
"description": "✅ Patterns learned elsewhere surface here automatically. ✅ Local only; no data leaves the machine. ❌ Could mix context if you later work on multiple client codebases."
|
||||
},
|
||||
{
|
||||
"label": "Project-scoped only",
|
||||
"description": "✅ Strict isolation between codebases on this machine. ✅ Nothing to audit later for cross-contamination. ❌ Every project starts learning from zero."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
},
|
||||
{
|
||||
"header": "Focus",
|
||||
"question": "D3 — Review all 7 design dimensions or focus on specific ones?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: I rated the plan 7/10. The biggest gaps are the five listed DESIGN.md violations: Save has no primary styling, section spacing is inconsistent, the error color fails WCAG AA, labels use three sizes, and Save shows no pending indicator. Mockups are skipped per your request. Your message already asked for all seven dimensions; confirming so I do not narrow scope.\nStakes if we pick wrong: narrowing skips passes where the plan could still hide gaps.\nRecommendation: A because you explicitly asked for all seven and the plan's remaining gaps span five of them.\nCompleteness: A=10/10, B=6/10.\nNet: full coverage vs a shorter session.",
|
||||
"options": [
|
||||
{
|
||||
"label": "All 7 dimensions (recommended)",
|
||||
"description": "✅ Every pass evaluated, each finding gets its own decision. ✅ Matches your request for all seven dimensions. ❌ More questions to answer: at least five remedy decisions."
|
||||
},
|
||||
{
|
||||
"label": "Focus on the 5 listed gaps only",
|
||||
"description": "✅ Shorter session centered on known violations. ✅ Still resolves each gap individually. ❌ Skips journey, AI-slop, and responsive passes where unlisted gaps may hide."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D1 — Add gstack skill routing rules to CLAUDE.md?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: gstack works best when your project's CLAUDE.md tells the agent which skill to invoke for which request (bugs → /investigate, ship → /ship). Without it you invoke skills by hand each time. Plan mode forbids editing CLAUDE.md now, so option A would be applied after this review exits plan mode.\nStakes if we pick wrong: mild; you lose auto-routing or gain a section you did not want in a fixture repo.\nRecommendation: A because routing is free once written and removable in one edit.\nNote: options differ in kind, not coverage — no completeness score.\nNet: convenience later vs a tidier CLAUDE.md now.": "Add routing rules (recommended)",
|
||||
"D2 — Enable cross-project learnings search?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: gstack stores small lessons per project. With this on, reviews can also read lessons from your other projects on this machine. Nothing leaves your machine. Good for solo developers; skip if you work across client codebases where mixing is a concern.\nStakes if we pick wrong: on for a multi-client machine could surface one client's patterns in another's review; off means slower compounding.\nRecommendation: A because this is a local, single-user sandbox with no client separation concern.\nNote: options differ in kind, not coverage — no completeness score.\nNet: faster compounding vs strict per-project isolation.": "Enable cross-project (recommended)",
|
||||
"D3 — Review all 7 design dimensions or focus on specific ones?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: I rated the plan 7/10. The biggest gaps are the five listed DESIGN.md violations: Save has no primary styling, section spacing is inconsistent, the error color fails WCAG AA, labels use three sizes, and Save shows no pending indicator. Mockups are skipped per your request. Your message already asked for all seven dimensions; confirming so I do not narrow scope.\nStakes if we pick wrong: narrowing skips passes where the plan could still hide gaps.\nRecommendation: A because you explicitly asked for all seven and the plan's remaining gaps span five of them.\nCompleteness: A=10/10, B=6/10.\nNet: full coverage vs a shorter session.": "All 7 dimensions (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:04:39.305Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "22820c07-b1f9-4c7c-a7b8-ba7acb607f01",
|
||||
"toolUseId": "toolu_01CstVkfwsmYknA3Cy12g8nA",
|
||||
"questions": [
|
||||
{
|
||||
"header": "Issue 1: Save",
|
||||
"question": "D4 — Issue 1 (F1): How should the header action group establish the primary action?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: Save, Reset, Cancel and Export currently look the same. Users scan, they do not read, so when four buttons shout equally the one they came for (Save) gets no priority. Hierarchy as service: what should they see first? DESIGN.md already answers it: Save is the only filled primary button, the other three are neutral ghost buttons.\nStakes if we pick wrong: users hesitate at the action row, or hit Reset/Export by mistake because nothing signals which is the commit action.\nRecommendation: 1A because it applies the existing DESIGN.md token with zero new visual exploration, which the plan forbids.\nCompleteness: 1A=10/10, 1B=9/10, 1C=2/10.\nNet: reuse the approved token vs invent extra separation vs leave the scan failure in place.",
|
||||
"options": [
|
||||
{
|
||||
"label": "1A: DESIGN.md token (recommended)",
|
||||
"description": "✅ Save becomes the only filled #1d4ed8 button with white text; Reset, Cancel, Export become neutral ghost buttons, exactly as DESIGN.md states. ✅ No new tokens, 44px geometry and DOM order unchanged. ❌ Export sits visually equal to Reset and Cancel even though it is not destructive. (human: ~1h / CC: ~5min)"
|
||||
},
|
||||
{
|
||||
"label": "1B: Token plus Export gap",
|
||||
"description": "✅ Same as 1A, and adds a spacer between Cancel and Export so the non-destructive action is visually grouped apart. ✅ Slightly clearer mental model of discard vs download. ❌ Introduces a layout token DESIGN.md does not define, contradicting the no-visual-exploration constraint. (human: ~2h / CC: ~10min)"
|
||||
},
|
||||
{
|
||||
"label": "1C: Keep all four identical",
|
||||
"description": "✅ Zero implementation change in this PR. ✅ No risk of styling regressions in Button. ❌ Ships a known DESIGN.md violation; the review score stays capped and users keep scanning a flat row."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D4 — Issue 1 (F1): How should the header action group establish the primary action?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: Save, Reset, Cancel and Export currently look the same. Users scan, they do not read, so when four buttons shout equally the one they came for (Save) gets no priority. Hierarchy as service: what should they see first? DESIGN.md already answers it: Save is the only filled primary button, the other three are neutral ghost buttons.\nStakes if we pick wrong: users hesitate at the action row, or hit Reset/Export by mistake because nothing signals which is the commit action.\nRecommendation: 1A because it applies the existing DESIGN.md token with zero new visual exploration, which the plan forbids.\nCompleteness: 1A=10/10, 1B=9/10, 1C=2/10.\nNet: reuse the approved token vs invent extra separation vs leave the scan failure in place.": "1A: DESIGN.md token (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:05:15.941Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "22820c07-b1f9-4c7c-a7b8-ba7acb607f01",
|
||||
"toolUseId": "toolu_01Sd6pzYGUVvycDqnHpdyeZd",
|
||||
"questions": [
|
||||
{
|
||||
"header": "Issue 2: Save",
|
||||
"question": "D5 — Issue 2 (F5): What does the user see during the 2-5 second Save request?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: After clicking Save nothing changes for several seconds, so users wonder if the click registered and may click again or leave. The plan already says pending feedback lives in the button and the status text must not repeat Saving. DESIGN.md defines the established pattern: an inline spinner beside the text Saving… inside the disabled Save button, aria-busy=true, honoring reduced motion. Export already uses this exact pattern.\nStakes if we pick wrong: duplicate submits, abandoned saves, or a skeleton that hides the values the user just typed.\nRecommendation: 2A because it is the existing pattern, matches Export, and keeps the user's typed values visible.\nCompleteness: 2A=10/10, 2B=5/10, 2C=1/10.\nNet: reuse the proven button-level pending pattern vs a page-level skeleton that hides edits vs a frozen page.",
|
||||
"options": [
|
||||
{
|
||||
"label": "2A: Spinner + Saving… in Save (recommended)",
|
||||
"description": "✅ Existing inline spinner beside Saving… inside Save, aria-disabled=true with activation guard, aria-busy=true, reduced-motion swaps spinner for static text. ✅ Identical to the Export pending pattern, so one component path. ❌ Button text width changes from Save to Saving…; reserve min-width so the row does not reflow. (human: ~2h / CC: ~10min)"
|
||||
},
|
||||
{
|
||||
"label": "2B: Skeleton overlay on the form",
|
||||
"description": "✅ Very visible page-level feedback that something is happening. ✅ Reuses the existing loading skeleton component. ❌ Hides the values the user just typed and conflicts with the accepted rule that unsaved values stay visible and focus never moves. (human: ~3h / CC: ~15min)"
|
||||
},
|
||||
{
|
||||
"label": "2C: Leave as is",
|
||||
"description": "✅ No change to Save in this PR. ✅ No risk of touching Button internals. ❌ Users see a frozen page for 2-5s and the plan keeps a documented DESIGN.md violation."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D5 — Issue 2 (F5): What does the user see during the 2-5 second Save request?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: After clicking Save nothing changes for several seconds, so users wonder if the click registered and may click again or leave. The plan already says pending feedback lives in the button and the status text must not repeat Saving. DESIGN.md defines the established pattern: an inline spinner beside the text Saving… inside the disabled Save button, aria-busy=true, honoring reduced motion. Export already uses this exact pattern.\nStakes if we pick wrong: duplicate submits, abandoned saves, or a skeleton that hides the values the user just typed.\nRecommendation: 2A because it is the existing pattern, matches Export, and keeps the user's typed values visible.\nCompleteness: 2A=10/10, 2B=5/10, 2C=1/10.\nNet: reuse the proven button-level pending pattern vs a page-level skeleton that hides edits vs a frozen page.": "2A: Spinner + Saving… in Save (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:06:13.146Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "22820c07-b1f9-4c7c-a7b8-ba7acb607f01",
|
||||
"toolUseId": "toolu_01LL2GZSrpDiP41VeZm9uC6n",
|
||||
"questions": [
|
||||
{
|
||||
"header": "Issue 3: Space",
|
||||
"question": "D6 — Issue 3 (F2): Which vertical spacing rhythm should the form use?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: Right now the gaps between sections are 24px in some places, 32px in others, 16px in a third. Gestalt proximity says spacing tells users what belongs together; random gaps make the grouping feel accidental. DESIGN.md defines an 8px base scale: 32px between sections, 24px between field groups, 8px from label to input.\nStakes if we pick wrong: the form reads as assembled rather than designed, and Profile vs Notifications grouping is visually ambiguous.\nRecommendation: 3A because it applies the existing DESIGN.md scale with no new tokens.\nCompleteness: 3A=10/10, 3B=7/10, 3C=2/10.\nNet: adopt the approved scale vs a tighter one-off scale vs keep the inconsistency.",
|
||||
"options": [
|
||||
{
|
||||
"label": "3A: DESIGN.md 8px scale (recommended)",
|
||||
"description": "✅ Sections 32px, field groups 24px, label-to-input 8px, applied to header, status, Profile and Notifications alike. ✅ More space above each h2 than below it, so headings attach to their fieldset. ❌ Slightly taller page at 320px; still no horizontal scroll. (human: ~1h / CC: ~5min)"
|
||||
},
|
||||
{
|
||||
"label": "3B: Compact 24/16/8 scale",
|
||||
"description": "✅ Denser form, less scrolling on small screens. ✅ Still a consistent 8px-based rhythm. ❌ Contradicts the checked-in DESIGN.md section token, creating two spacing systems in one app. (human: ~1h / CC: ~5min)"
|
||||
},
|
||||
{
|
||||
"label": "3C: Leave as is",
|
||||
"description": "✅ No layout change in this PR. ✅ Zero regression risk in shared spacing utilities. ❌ Keeps a documented DESIGN.md violation and an inconsistent rhythm users can feel."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D6 — Issue 3 (F2): Which vertical spacing rhythm should the form use?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: Right now the gaps between sections are 24px in some places, 32px in others, 16px in a third. Gestalt proximity says spacing tells users what belongs together; random gaps make the grouping feel accidental. DESIGN.md defines an 8px base scale: 32px between sections, 24px between field groups, 8px from label to input.\nStakes if we pick wrong: the form reads as assembled rather than designed, and Profile vs Notifications grouping is visually ambiguous.\nRecommendation: 3A because it applies the existing DESIGN.md scale with no new tokens.\nCompleteness: 3A=10/10, 3B=7/10, 3C=2/10.\nNet: adopt the approved scale vs a tighter one-off scale vs keep the inconsistency.": "3A: DESIGN.md 8px scale (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:07:54.092Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "22820c07-b1f9-4c7c-a7b8-ba7acb607f01",
|
||||
"toolUseId": "toolu_0183dhn4woP6Do6yCmBrYkCj",
|
||||
"questions": [
|
||||
{
|
||||
"header": "Issue 4: Type",
|
||||
"question": "D7 — Issue 4 (F4): Which type scale should form labels and headings use?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: Labels currently appear at 14px, 16px and 18px with no rule for which is which. Three sizes for one role flattens hierarchy and makes 14px labels the smallest text on the page, which also fails the never-below-16px body rule. DESIGN.md defines two roles: 16px for body, form labels and helper text; 20px for the Profile and Notifications headings.\nStakes if we pick wrong: labels compete with headings, 14px text is hard to read on mobile, and the page reads as three different forms stitched together.\nRecommendation: 4A because it applies the two DESIGN.md roles directly and removes the sub-16px text.\nCompleteness: 4A=10/10, 4B=6/10, 4C=2/10.\nNet: two clear roles vs keep a small helper size vs three arbitrary sizes.",
|
||||
"options": [
|
||||
{
|
||||
"label": "4A: Two roles, 16px / 20px (recommended)",
|
||||
"description": "✅ All labels, helper text, status text, button text and body at 16px; h2 section headings at 20px; h1 keeps its existing app size. ✅ Eliminates 14px text so nothing on the page falls below the 16px floor. ❌ Helper text loses its smaller visual distinction; rely on color-agnostic wording and spacing instead. (human: ~1h / CC: ~5min)"
|
||||
},
|
||||
{
|
||||
"label": "4B: 16/20 plus 14px helper text",
|
||||
"description": "✅ Labels and headings match DESIGN.md roles. ✅ Helper text stays visually subordinate to labels. ❌ Keeps sub-16px body text, violating both DESIGN.md's two-role rule and the universal small-type rule. (human: ~1h / CC: ~5min)"
|
||||
},
|
||||
{
|
||||
"label": "4C: Leave as is",
|
||||
"description": "✅ No CSS change in this PR. ✅ No risk in shared Field styles. ❌ Three label sizes persist, hierarchy stays flat, and 14px text remains on mobile."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D7 — Issue 4 (F4): Which type scale should form labels and headings use?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: Labels currently appear at 14px, 16px and 18px with no rule for which is which. Three sizes for one role flattens hierarchy and makes 14px labels the smallest text on the page, which also fails the never-below-16px body rule. DESIGN.md defines two roles: 16px for body, form labels and helper text; 20px for the Profile and Notifications headings.\nStakes if we pick wrong: labels compete with headings, 14px text is hard to read on mobile, and the page reads as three different forms stitched together.\nRecommendation: 4A because it applies the two DESIGN.md roles directly and removes the sub-16px text.\nCompleteness: 4A=10/10, 4B=6/10, 4C=2/10.\nNet: two clear roles vs keep a small helper size vs three arbitrary sizes.": "4A: Two roles, 16px / 20px (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:08:29.247Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "22820c07-b1f9-4c7c-a7b8-ba7acb607f01",
|
||||
"toolUseId": "toolu_011zzhbj36f5yXSkqTLYTfw7",
|
||||
"questions": [
|
||||
{
|
||||
"header": "Issue 5: Error",
|
||||
"question": "D8 — Issue 5 (F3): Which error color treatment should the plan specify?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: WCAG AA (the Web Content Accessibility Guidelines level most products target) requires normal text to contrast at least 4.5:1 against its background. The current red-on-light-pink error text sits near 3:1, so people with low vision or a dim screen struggle to read exactly the message that tells them what went wrong. DESIGN.md defines the fix: error text #991b1b on error surface #fef2f2, which measures about 7.6:1, always paired with an icon and explicit text so color is never the only signal.\nStakes if we pick wrong: an unreadable error at the most stressful step, plus a legal-exposure accessibility failure.\nRecommendation: 5A because it applies the existing DESIGN.md tokens, clears AA with margin, and already includes the icon plus text requirement.\nCompleteness: 5A=10/10, 5B=6/10, 5C=1/10.\nNet: approved tokens with icon vs a darker one-off red vs keep the AA failure.",
|
||||
"options": [
|
||||
{
|
||||
"label": "5A: DESIGN.md error tokens (recommended)",
|
||||
"description": "✅ error.text #991b1b on error.surface #fef2f2 (~7.6:1), with the existing error icon and explicit text, applied to field errors, ErrorSummary, and the network/Export error area. ✅ Status never communicated by color alone, matching the inherited rule. ❌ Retry button beside the error must also be checked against #fef2f2 for its own 4.5:1 and 3:1 focus-ring contrast. (human: ~1h / CC: ~5min)"
|
||||
},
|
||||
{
|
||||
"label": "5B: Darken current red only",
|
||||
"description": "✅ Minimal CSS change: pick a red that reaches 4.5:1 on the current pink. ✅ Keeps the current look mostly intact. ❌ Creates a third red in the app and leaves the surface off-token; DESIGN.md already names the pair. (human: ~30min / CC: ~5min)"
|
||||
},
|
||||
{
|
||||
"label": "5C: Leave as is",
|
||||
"description": "✅ No change in this PR. ✅ No shared token edits. ❌ Ships a WCAG AA failure on the error message and keeps a documented DESIGN.md violation."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D8 — Issue 5 (F3): Which error color treatment should the plan specify?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: WCAG AA (the Web Content Accessibility Guidelines level most products target) requires normal text to contrast at least 4.5:1 against its background. The current red-on-light-pink error text sits near 3:1, so people with low vision or a dim screen struggle to read exactly the message that tells them what went wrong. DESIGN.md defines the fix: error text #991b1b on error surface #fef2f2, which measures about 7.6:1, always paired with an icon and explicit text so color is never the only signal.\nStakes if we pick wrong: an unreadable error at the most stressful step, plus a legal-exposure accessibility failure.\nRecommendation: 5A because it applies the existing DESIGN.md tokens, clears AA with margin, and already includes the icon plus text requirement.\nCompleteness: 5A=10/10, 5B=6/10, 5C=1/10.\nNet: approved tokens with icon vs a darker one-off red vs keep the AA failure.": "5A: DESIGN.md error tokens (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:09:15.978Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "22820c07-b1f9-4c7c-a7b8-ba7acb607f01",
|
||||
"toolUseId": "toolu_019GjHri3qXZtWZH8VJhAr2n",
|
||||
"questions": [
|
||||
{
|
||||
"header": "Issue 6: 320px",
|
||||
"question": "D9 — Issue 6: How do the status text and its Retry button lay out when they do not fit on one line at 320px?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: The plan says Retry sits beside the status text. On a 320px phone an error sentence plus a 44px button will not fit on one line, and the plan does not say what happens. Left undecided, an engineer picks one of three behaviors at random: the text truncates, the row overflows and causes horizontal scroll, or the button wraps somewhere. Only the last keeps the accepted no-horizontal-scroll rule.\nStakes if we pick wrong: a truncated error message the user cannot read, or a horizontal scrollbar on the one page that promised none.\nRecommendation: 6A because wrapping keeps the full message readable, preserves the 44px target, and needs no new component.\nCompleteness: 6A=10/10, 6B=7/10, 6C=3/10.\nNet: full message and full-width button vs a fixed side column vs one-line truncation.",
|
||||
"options": [
|
||||
{
|
||||
"label": "6A: Wrap Retry below the text (recommended)",
|
||||
"description": "✅ Status text wraps to as many lines as needed; when the row cannot fit, Retry drops to its own line below, left-aligned, 44px tall, 8px gap, DOM order unchanged. ✅ No truncation and no horizontal scroll at 320px; identical behavior for save, export and load errors. ❌ The status area grows taller during errors, pushing fields down slightly. (human: ~1h / CC: ~5min)"
|
||||
},
|
||||
{
|
||||
"label": "6B: Fixed-width Retry column",
|
||||
"description": "✅ Retry always visible at the right edge, same position at every width. ✅ Status area height stays predictable. ❌ Leaves roughly 220px for text at 320px, so longer messages wrap heavily beside a tall button and layout looks lopsided. (human: ~1h / CC: ~5min)"
|
||||
},
|
||||
{
|
||||
"label": "6C: Single line, truncate text",
|
||||
"description": "✅ Simplest CSS; the row never changes height. ✅ Retry stays beside the text always. ❌ Users cannot read the full error at 320px, and truncating a live-region message also cuts what screen readers announce visually vs audibly. (human: ~30min / CC: ~5min)"
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D9 — Issue 6: How do the status text and its Retry button lay out when they do not fit on one line at 320px?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: The plan says Retry sits beside the status text. On a 320px phone an error sentence plus a 44px button will not fit on one line, and the plan does not say what happens. Left undecided, an engineer picks one of three behaviors at random: the text truncates, the row overflows and causes horizontal scroll, or the button wraps somewhere. Only the last keeps the accepted no-horizontal-scroll rule.\nStakes if we pick wrong: a truncated error message the user cannot read, or a horizontal scrollbar on the one page that promised none.\nRecommendation: 6A because wrapping keeps the full message readable, preserves the 44px target, and needs no new component.\nCompleteness: 6A=10/10, 6B=7/10, 6C=3/10.\nNet: full message and full-width button vs a fixed side column vs one-line truncation.": "6A: Wrap Retry below the text (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:10:27.866Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "22820c07-b1f9-4c7c-a7b8-ba7acb607f01",
|
||||
"toolUseId": "toolu_01ETdnJuZB7ZunqCk5e3GRht",
|
||||
"questions": [
|
||||
{
|
||||
"header": "TODO: Font",
|
||||
"question": "D10 — TODO proposal: track an app-wide typeface decision as deferred design debt?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: The plan keeps system-ui as the app font, which is the right call for this form update because changing a font is an app-wide decision, not a settings-page one. But system-ui is the single strongest tell that nobody chose typography. This is not a change to this plan; it is whether to write the debt down so someone revisits it deliberately.\nWhat: Choose a real body/UI typeface app-wide and update DESIGN.md. Why: system-ui reads as unfinished and belongs to no design system. Pros: one decision lifts every screen. Cons: font loading cost, cross-platform QA, a DESIGN.md revision. Context: surfaced by /plan-design-review Pass 4 on 2026-09-10; the settings plan explicitly forbids visual exploration so it was not changed here. Depends on: a /design-consultation or brand decision.\nStakes if we pick wrong: the debt is forgotten, or this PR grows into a font migration.\nRecommendation: A because writing it down costs nothing and keeps this PR scoped.\nNote: options differ in kind, not coverage — no completeness score.\nNet: remember the debt vs drop it vs expand this PR.",
|
||||
"options": [
|
||||
{
|
||||
"label": "A: Add to TODOS.md (recommended)",
|
||||
"description": "✅ The debt is recorded with context so a future session can act deliberately. ✅ This PR stays scoped to the settings form as the plan requires. ❌ TODOS.md does not exist yet in this repo; the file is created after plan mode ends. (human: ~5min / CC: ~1min)"
|
||||
},
|
||||
{
|
||||
"label": "B: Skip, not valuable enough",
|
||||
"description": "✅ No new file or backlog item for a fixture repo. ✅ Zero follow-up noise. ❌ The typography tell is forgotten until someone notices it again."
|
||||
},
|
||||
{
|
||||
"label": "C: Build it now in this PR",
|
||||
"description": "✅ Fixes the strongest AI-slop tell immediately. ✅ One less future migration. ❌ Contradicts the plan's explicit constraint to retain system-ui and avoid visual exploration; touches every screen in the app. (human: ~2 days / CC: ~1h)"
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D10 — TODO proposal: track an app-wide typeface decision as deferred design debt?\nProject/branch/task: gstack-plan-count-xG5e2f on main, reviewing PLAN.md.\nELI10: The plan keeps system-ui as the app font, which is the right call for this form update because changing a font is an app-wide decision, not a settings-page one. But system-ui is the single strongest tell that nobody chose typography. This is not a change to this plan; it is whether to write the debt down so someone revisits it deliberately.\nWhat: Choose a real body/UI typeface app-wide and update DESIGN.md. Why: system-ui reads as unfinished and belongs to no design system. Pros: one decision lifts every screen. Cons: font loading cost, cross-platform QA, a DESIGN.md revision. Context: surfaced by /plan-design-review Pass 4 on 2026-09-10; the settings plan explicitly forbids visual exploration so it was not changed here. Depends on: a /design-consultation or brand decision.\nStakes if we pick wrong: the debt is forgotten, or this PR grows into a font migration.\nRecommendation: A because writing it down costs nothing and keeps this PR scoped.\nNote: options differ in kind, not coverage — no completeness score.\nNet: remember the debt vs drop it vs expand this PR.": "A: Add to TODOS.md (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:11:09.095Z"
|
||||
}
|
||||
],
|
||||
"retryCalls": [
|
||||
{
|
||||
"sessionId": "d5bc63d3-0857-480c-9e8d-176a00bf4fdf",
|
||||
"toolUseId": "toolu_01WYKYudReox1LyBVhwazCvp",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D1 — Add gstack skill routing rules to this project's CLAUDE.md?\nProject/branch/task: main branch, plan-design-review of PLAN.md (Settings Page UI redesign).\nELI10: gstack works best when the project's CLAUDE.md includes skill routing rules, so future requests like \"review this design\" or \"ship it\" get routed to the matching skill automatically. This is a one-time setup prompt per project. Note: plan mode is active, so if you pick A the CLAUDE.md edit and commit happen after the review finishes and plan mode exits, not now.\nStakes if we pick wrong: Picking A on a throwaway fixture repo adds a commit you may not want; picking B means you invoke skills manually here (re-enable anytime via gstack-config).\nRecommendation: A because routing rules make later skill invocations automatic, and the append is small and reversible.\nNote: options differ in kind, not coverage — no completeness score.\nNet: a small convenience commit versus keeping this repo untouched beyond the plan review.",
|
||||
"header": "Routing",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Add routing rules (recommended)",
|
||||
"description": "✅ Future requests auto-route to the right gstack skill without typing slash commands. ✅ Append is a short section at the end of CLAUDE.md, easy to revert. ❌ Adds a commit to this repo after plan mode exits, which may be unwanted on a fixture."
|
||||
},
|
||||
{
|
||||
"label": "No thanks, manual invocation",
|
||||
"description": "✅ Repo stays untouched apart from the plan file you asked for. ✅ You can re-enable later with gstack-config set routing_declined false. ❌ You must type /skill-name yourself for each gstack workflow in this project."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D1 — Add gstack skill routing rules to this project's CLAUDE.md?\nProject/branch/task: main branch, plan-design-review of PLAN.md (Settings Page UI redesign).\nELI10: gstack works best when the project's CLAUDE.md includes skill routing rules, so future requests like \"review this design\" or \"ship it\" get routed to the matching skill automatically. This is a one-time setup prompt per project. Note: plan mode is active, so if you pick A the CLAUDE.md edit and commit happen after the review finishes and plan mode exits, not now.\nStakes if we pick wrong: Picking A on a throwaway fixture repo adds a commit you may not want; picking B means you invoke skills manually here (re-enable anytime via gstack-config).\nRecommendation: A because routing rules make later skill invocations automatic, and the append is small and reversible.\nNote: options differ in kind, not coverage — no completeness score.\nNet: a small convenience commit versus keeping this repo untouched beyond the plan review.": "Add routing rules (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:16:40.262Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "d5bc63d3-0857-480c-9e8d-176a00bf4fdf",
|
||||
"toolUseId": "toolu_01FNEUWRFeLzxoFCnWp7BQAT",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D2 — Review all seven design dimensions, or focus on specific ones?\nProject/branch/task: main, plan-design-review of PLAN.md (Account settings form update).\nELI10: I rated the plan 6/10 on design completeness. The biggest gaps are the undifferentiated Save button, the frozen 2-5 second save with no feedback, and the error styling below WCAG AA. You asked for all seven dimensions and text-only; this confirms scope before I start the passes.\nStakes if we pick wrong: Narrowing skips passes where the listed gaps live (spacing and typography sit in Design System, error contrast in Accessibility), leaving them unreviewed.\nRecommendation: A because you asked for all seven and the five gaps spread across five different passes.\nCompleteness: A=10/10, B=6/10\nNet: full coverage in one sitting versus a faster pass that leaves some listed gaps undecided.",
|
||||
"header": "Focus",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "All seven dimensions (recommended)",
|
||||
"description": "✅ Every listed gap gets its own decision in the pass where it belongs. ✅ State table, storyboard, and decision register all land in the plan. ❌ Around eight decision prompts before the review closes."
|
||||
},
|
||||
{
|
||||
"label": "Focus on the five listed gaps only",
|
||||
"description": "✅ Fewer prompts; skips journey storyboard and AI-slop evaluation. ✅ Still resolves the five documented deviations from DESIGN.md. ❌ Passes 3 and 4 go unevaluated, so the completion summary cannot call the plan design-complete."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D2 — Review all seven design dimensions, or focus on specific ones?\nProject/branch/task: main, plan-design-review of PLAN.md (Account settings form update).\nELI10: I rated the plan 6/10 on design completeness. The biggest gaps are the undifferentiated Save button, the frozen 2-5 second save with no feedback, and the error styling below WCAG AA. You asked for all seven dimensions and text-only; this confirms scope before I start the passes.\nStakes if we pick wrong: Narrowing skips passes where the listed gaps live (spacing and typography sit in Design System, error contrast in Accessibility), leaving them unreviewed.\nRecommendation: A because you asked for all seven and the five gaps spread across five different passes.\nCompleteness: A=10/10, B=6/10\nNet: full coverage in one sitting versus a faster pass that leaves some listed gaps undecided.": "All seven dimensions (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:19:26.796Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "d5bc63d3-0857-480c-9e8d-176a00bf4fdf",
|
||||
"toolUseId": "toolu_018VpEseShcTk9u1jb13wN9P",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D3 — Let gstack search learnings from your other local projects?\nProject/branch/task: main, plan-design-review of PLAN.md.\nELI10: gstack records small lessons per project (pitfalls, patterns). It can also search lessons from your other projects on this machine to spot patterns that apply here. Everything stays local; nothing leaves your machine. This is a one-time setting.\nStakes if we pick wrong: Enabling on a machine with multiple client codebases could surface one client's conventions while working on another; disabling means each project learns from scratch.\nRecommendation: A because this is a local-only lookup and most solo setups benefit from shared lessons.\nNote: options differ in kind, not coverage — no completeness score.\nNet: shared local lessons versus strict per-project isolation.",
|
||||
"header": "Learnings",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Enable cross-project (recommended)",
|
||||
"description": "✅ Past lessons from other local projects inform this review and future ones. ✅ Stays on this machine; no network, no upload. ❌ Could surface conventions from an unrelated client codebase if you work across several."
|
||||
},
|
||||
{
|
||||
"label": "Keep project-scoped only",
|
||||
"description": "✅ Strict isolation between projects; nothing crosses over. ✅ Safe default for multi-client machines. ❌ This project starts with zero prior learnings and builds them alone."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D3 — Let gstack search learnings from your other local projects?\nProject/branch/task: main, plan-design-review of PLAN.md.\nELI10: gstack records small lessons per project (pitfalls, patterns). It can also search lessons from your other projects on this machine to spot patterns that apply here. Everything stays local; nothing leaves your machine. This is a one-time setting.\nStakes if we pick wrong: Enabling on a machine with multiple client codebases could surface one client's conventions while working on another; disabling means each project learns from scratch.\nRecommendation: A because this is a local-only lookup and most solo setups benefit from shared lessons.\nNote: options differ in kind, not coverage — no completeness score.\nNet: shared local lessons versus strict per-project isolation.": "Enable cross-project (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:19:42.903Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "d5bc63d3-0857-480c-9e8d-176a00bf4fdf",
|
||||
"toolUseId": "toolu_01Vn8BRWP4G281vP2iVaXngy",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D4 — Issue 1 (Pass 1): how should the plan make Save the visible primary action?\nProject/branch/task: main, PLAN.md gap \"Visual Hierarchy\": Save renders identical to Reset, Cancel, Export.\nELI10: Users scan, they don't read. Four same-looking buttons force a pause at the exact moment they want to commit their edits. DESIGN.md already names the answer: Save is the only filled button (#1d4ed8 with white text, 6.7:1 contrast), the other three are neutral ghost buttons. Principle: hierarchy as service, make the right choice the most visible choice.\nStakes if we pick wrong: Mis-clicks on Reset or Cancel next to Save trigger discard dialogs mid-task; users lose confidence that their edit will stick.\nRecommendation: 1A because it is the approved token, touches only Button variants, and keeps the 44px geometry and focus ring unchanged.\nCompleteness: 1A=10/10, 1B=7/10, 1C=3/10\nNet: apply the approved primary token now, or leave the implementer to invent emphasis.",
|
||||
"header": "Issue 1",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "1A Filled Save, ghost secondaries (recommended)",
|
||||
"description": "✅ Uses the DESIGN.md primary token; Save reads as primary at a glance on every viewport. ✅ Focus ring stays visible: 2px offset leaves a white gap between ring and fill. (human: ~1h / CC: ~5min) ❌ Reset, Cancel and Export now look alike, so their labels carry the distinction alone."
|
||||
},
|
||||
{
|
||||
"label": "1B Filled Save plus visually separate Export",
|
||||
"description": "✅ Save is primary and Export gets extra spacing or a divider so utility is separated from form actions. ✅ Reduces the chance of hitting Export while aiming for Cancel. (human: ~2h / CC: ~10min) ❌ Adds layout change to the header beyond the approved structure and complicates the 640px equal-column row."
|
||||
},
|
||||
{
|
||||
"label": "1C Keep four equal buttons",
|
||||
"description": "✅ Zero styling work; relies on Save being first in order. ✅ No risk of token mismatch. ❌ Leaves the documented gap open; position alone fails the scan test, so the pass stays at 6/10."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D4 — Issue 1 (Pass 1): how should the plan make Save the visible primary action?\nProject/branch/task: main, PLAN.md gap \"Visual Hierarchy\": Save renders identical to Reset, Cancel, Export.\nELI10: Users scan, they don't read. Four same-looking buttons force a pause at the exact moment they want to commit their edits. DESIGN.md already names the answer: Save is the only filled button (#1d4ed8 with white text, 6.7:1 contrast), the other three are neutral ghost buttons. Principle: hierarchy as service, make the right choice the most visible choice.\nStakes if we pick wrong: Mis-clicks on Reset or Cancel next to Save trigger discard dialogs mid-task; users lose confidence that their edit will stick.\nRecommendation: 1A because it is the approved token, touches only Button variants, and keeps the 44px geometry and focus ring unchanged.\nCompleteness: 1A=10/10, 1B=7/10, 1C=3/10\nNet: apply the approved primary token now, or leave the implementer to invent emphasis.": "1A Filled Save, ghost secondaries (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:20:13.115Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "d5bc63d3-0857-480c-9e8d-176a00bf4fdf",
|
||||
"toolUseId": "toolu_019Gzp7QySwkcA2zDNGLGy8c",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D5 — Issue 2 (Pass 2): what does the user see while Save is pending for 2-5 seconds?\nProject/branch/task: main, PLAN.md gap \"Motion\": Save has no loading indicator; page looks frozen.\nELI10: The plan already blocks repeat submits, but nothing tells the user the click landed. DESIGN.md has an established pending pattern: an inline spinner beside the text \"Saving…\" inside the Save button, aria-busy=true, reduced-motion support, while InlineStatus keeps its current text. The PLAN.md gap mentions \"spinner or skeleton\"; a skeleton would hide the user's own edits during save, which conflicts with preserving unsaved values. Principle: visibility of system status (Nielsen), feedback within one second.\nStakes if we pick wrong: Users double-click, assume a hang, or navigate away mid-save; the atomic-save guarantee is invisible so trust erodes at the exact moment it should build.\nRecommendation: 2A because it is the approved pattern, Export already uses the identical mechanics, and it keeps the live region quiet.\nCompleteness: 2A=10/10, 2B=5/10, 2C=6/10\nNet: reuse the approved button-level pending pattern, or invent a page-level one that fights the accepted state rules.",
|
||||
"header": "Issue 2",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "2A Spinner + \"Saving…\" in the Save button (recommended)",
|
||||
"description": "✅ Matches DESIGN.md and mirrors the existing Export pending pattern exactly, one implementation for both. ✅ Feedback is at the point of action; aria-busy announces it without repeating text in the status live region. (human: ~2h / CC: ~10min) ❌ Button width shifts slightly as text changes unless min-width is reserved; note it in the task."
|
||||
},
|
||||
{
|
||||
"label": "2B Skeleton or overlay over the form while saving",
|
||||
"description": "✅ Very obvious that the page is busy. ✅ No per-button work. ❌ Hides the user's unsaved values and the InlineStatus during save, contradicting the accepted rule that fields and status stay visible and unchanged while pending."
|
||||
},
|
||||
{
|
||||
"label": "2C Spinner in button plus \"Saving…\" in InlineStatus",
|
||||
"description": "✅ Redundant feedback for users who don't look at the button. ✅ Still keeps fields visible. ❌ Directly violates the accepted rule: pending feedback belongs to the button, do not repeat Saving… in the status live region; also double-announces to screen readers."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D5 — Issue 2 (Pass 2): what does the user see while Save is pending for 2-5 seconds?\nProject/branch/task: main, PLAN.md gap \"Motion\": Save has no loading indicator; page looks frozen.\nELI10: The plan already blocks repeat submits, but nothing tells the user the click landed. DESIGN.md has an established pending pattern: an inline spinner beside the text \"Saving…\" inside the Save button, aria-busy=true, reduced-motion support, while InlineStatus keeps its current text. The PLAN.md gap mentions \"spinner or skeleton\"; a skeleton would hide the user's own edits during save, which conflicts with preserving unsaved values. Principle: visibility of system status (Nielsen), feedback within one second.\nStakes if we pick wrong: Users double-click, assume a hang, or navigate away mid-save; the atomic-save guarantee is invisible so trust erodes at the exact moment it should build.\nRecommendation: 2A because it is the approved pattern, Export already uses the identical mechanics, and it keeps the live region quiet.\nCompleteness: 2A=10/10, 2B=5/10, 2C=6/10\nNet: reuse the approved button-level pending pattern, or invent a page-level one that fights the accepted state rules.": "2A Spinner + \"Saving…\" in the Save button (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:20:54.388Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "d5bc63d3-0857-480c-9e8d-176a00bf4fdf",
|
||||
"toolUseId": "toolu_01L8FvX7AVMVEevwV8HJ5w4w",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D6 — Issue 3 (Pass 5): how should the plan fix the 24/32/16px section spacing?\nProject/branch/task: main, PLAN.md gap \"Spacing\": no consistent vertical rhythm between sections.\nELI10: DESIGN.md already defines an 8px base with three levels: 32px between sections (Profile to Notifications, header to form), 24px between field groups inside a section, 8px from a label to its input. The current form uses 24 and 32 interchangeably at the section level and 16 once, which reads as accidental. Principle: Gestalt proximity, spacing tells the user what belongs together.\nStakes if we pick wrong: Sections blur into each other at 320px where whitespace is the only separator besides the h2, and the form reads as assembled rather than designed.\nRecommendation: 3A because it maps each gap to the approved level with no new tokens and is a stylesheet-only change.\nCompleteness: 3A=10/10, 3B=6/10, 3C=3/10\nNet: adopt the approved three-level rhythm, or pick one flat value that erases the section versus field distinction.",
|
||||
"header": "Issue 3",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "3A Apply the 8px scale: 32 / 24 / 8 (recommended)",
|
||||
"description": "✅ Sections 32px, field groups 24px, label-to-input 8px, exactly the DESIGN.md tokens; the 16px outlier is removed. ✅ Rhythm survives the 640px stack because the same tokens apply at every width. (human: ~1h / CC: ~5min) ❌ Requires auditing every margin on the page, including header-to-actions and actions-to-status."
|
||||
},
|
||||
{
|
||||
"label": "3B Single 24px gap everywhere",
|
||||
"description": "✅ Trivially consistent; one value to check. ✅ Fits 320px with slightly less scroll. ❌ Flattens the hierarchy: a field group and a new section look equally spaced, so the h2 does all the separation work."
|
||||
},
|
||||
{
|
||||
"label": "3C Leave spacing as is",
|
||||
"description": "✅ No work now. ✅ No regression risk in existing snapshots. ❌ Documented deviation stays open; Pass 5 cannot rise above 6/10."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D6 — Issue 3 (Pass 5): how should the plan fix the 24/32/16px section spacing?\nProject/branch/task: main, PLAN.md gap \"Spacing\": no consistent vertical rhythm between sections.\nELI10: DESIGN.md already defines an 8px base with three levels: 32px between sections (Profile to Notifications, header to form), 24px between field groups inside a section, 8px from a label to its input. The current form uses 24 and 32 interchangeably at the section level and 16 once, which reads as accidental. Principle: Gestalt proximity, spacing tells the user what belongs together.\nStakes if we pick wrong: Sections blur into each other at 320px where whitespace is the only separator besides the h2, and the form reads as assembled rather than designed.\nRecommendation: 3A because it maps each gap to the approved level with no new tokens and is a stylesheet-only change.\nCompleteness: 3A=10/10, 3B=6/10, 3C=3/10\nNet: adopt the approved three-level rhythm, or pick one flat value that erases the section versus field distinction.": "3A Apply the 8px scale: 32 / 24 / 8 (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:22:24.176Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "d5bc63d3-0857-480c-9e8d-176a00bf4fdf",
|
||||
"toolUseId": "toolu_0141v4THyS2kQGE37ANroPHS",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D7 — Issue 4 (Pass 5): how should the plan collapse the 14/16/18px label sizes?\nProject/branch/task: main, PLAN.md gap \"Typography\": three label sizes across the form; two would create stronger hierarchy.\nELI10: DESIGN.md defines exactly two type roles: 16px for body, form labels and helper text, and 20px for the Profile and Notifications h2 headings. The current 14/16/18 mix has two problems: 14px labels sit below the 16px body-text floor for readability, and 18px labels compete with the headings so the section structure blurs. Principle: hierarchy as service, and the universal rule against body text under 16px.\nStakes if we pick wrong: Small labels hurt legibility for anyone over 40 or on a phone in sunlight; near-equal sizes make Profile and Notifications look like one long list.\nRecommendation: 4A because it is the approved two-role scale, removes the sub-16px labels, and lets the h2 headings carry structure alone.\nCompleteness: 4A=10/10, 4B=7/10, 4C=3/10\nNet: adopt the approved two-role scale, or keep a middle size that reintroduces the competition the gap describes.",
|
||||
"header": "Issue 4",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "4A Two roles: 16px labels/helper, 20px h2 (recommended)",
|
||||
"description": "✅ Exactly the DESIGN.md scale; no label falls below 16px, so the sub-AA readability risk disappears. ✅ Clears the flat-type-hierarchy finding in Pass 4 and the Pass 1 remainder in one change. (human: ~1h / CC: ~5min) ❌ Helper text and labels share a size, so helper text must be distinguished by color token and position rather than size."
|
||||
},
|
||||
{
|
||||
"label": "4B Three roles: 16px labels, 14px helper, 20px h2",
|
||||
"description": "✅ Helper text visibly subordinate to its label. ✅ Common convention users recognize. ❌ Reintroduces sub-16px text and a third role DESIGN.md does not define, so it needs a new token and a contrast check."
|
||||
},
|
||||
{
|
||||
"label": "4C Keep 14/16/18",
|
||||
"description": "✅ No stylesheet change. ✅ No visual regression on existing screenshots. ❌ Documented deviation stays open; Pass 4 and Pass 5 stay capped and the 14px labels remain below the readability floor."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D7 — Issue 4 (Pass 5): how should the plan collapse the 14/16/18px label sizes?\nProject/branch/task: main, PLAN.md gap \"Typography\": three label sizes across the form; two would create stronger hierarchy.\nELI10: DESIGN.md defines exactly two type roles: 16px for body, form labels and helper text, and 20px for the Profile and Notifications h2 headings. The current 14/16/18 mix has two problems: 14px labels sit below the 16px body-text floor for readability, and 18px labels compete with the headings so the section structure blurs. Principle: hierarchy as service, and the universal rule against body text under 16px.\nStakes if we pick wrong: Small labels hurt legibility for anyone over 40 or on a phone in sunlight; near-equal sizes make Profile and Notifications look like one long list.\nRecommendation: 4A because it is the approved two-role scale, removes the sub-16px labels, and lets the h2 headings carry structure alone.\nCompleteness: 4A=10/10, 4B=7/10, 4C=3/10\nNet: adopt the approved two-role scale, or keep a middle size that reintroduces the competition the gap describes.": "4A Two roles: 16px labels/helper, 20px h2 (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:22:53.893Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "d5bc63d3-0857-480c-9e8d-176a00bf4fdf",
|
||||
"toolUseId": "toolu_0124HzFi6vVFGUxkf37wCp2w",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D8 — Issue 5 (Pass 6): how should the plan fix the error message contrast?\nProject/branch/task: main, PLAN.md gap \"Color\": red text on light pink at roughly 3:1, below WCAG AA (4.5:1 for text).\nELI10: The moment a save fails is when the user most needs to read the message, and right now it is the least legible text on the page. DESIGN.md already specifies the fix: error text #991b1b on error surface #fef2f2, which measures 7.6:1 (passes AA and AAA), paired with an icon and explicit text so the state never depends on color alone. Principle: accessibility is not optional; trust is earned at the pixel level.\nStakes if we pick wrong: Low-vision users and anyone on a dim or glare-hit screen cannot read why their save failed, so they retry blindly or abandon with edits unsaved.\nRecommendation: 5A because it is the approved token pair with measured headroom, and applies to field errors, the ErrorSummary, and the network/export error area in one change.\nCompleteness: 5A=10/10, 5B=7/10, 5C=2/10\nNet: adopt the approved 7.6:1 pair everywhere errors render, or patch text color alone and leave the surface undefined.",
|
||||
"header": "Issue 5",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "5A #991b1b on #fef2f2 + icon + explicit text (recommended)",
|
||||
"description": "✅ 7.6:1 contrast clears AA and AAA; icon is decorative (aria-hidden) with the text carrying meaning, so it works for colorblind users and screen readers. ✅ One token pair applied to field errors, ErrorSummary, and the network/export error area. (human: ~1h / CC: ~5min) ❌ Every error surface on the page needs a contrast re-measure after the change to confirm no override slips through."
|
||||
},
|
||||
{
|
||||
"label": "5B Darken text only, keep current pink surface",
|
||||
"description": "✅ Smaller diff; only the text token changes. ✅ Likely reaches 4.5:1 depending on the existing pink. ❌ Surface stays an undocumented color outside DESIGN.md, and the contrast must be measured per instance rather than known from the token pair."
|
||||
},
|
||||
{
|
||||
"label": "5C Leave as is",
|
||||
"description": "✅ No work. ✅ No visual change to existing screenshots. ❌ Ships a known WCAG AA failure on the most important text in the flow; Pass 6 stays at 7/10 and Pass 3 stays at 8/10."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D8 — Issue 5 (Pass 6): how should the plan fix the error message contrast?\nProject/branch/task: main, PLAN.md gap \"Color\": red text on light pink at roughly 3:1, below WCAG AA (4.5:1 for text).\nELI10: The moment a save fails is when the user most needs to read the message, and right now it is the least legible text on the page. DESIGN.md already specifies the fix: error text #991b1b on error surface #fef2f2, which measures 7.6:1 (passes AA and AAA), paired with an icon and explicit text so the state never depends on color alone. Principle: accessibility is not optional; trust is earned at the pixel level.\nStakes if we pick wrong: Low-vision users and anyone on a dim or glare-hit screen cannot read why their save failed, so they retry blindly or abandon with edits unsaved.\nRecommendation: 5A because it is the approved token pair with measured headroom, and applies to field errors, the ErrorSummary, and the network/export error area in one change.\nCompleteness: 5A=10/10, 5B=7/10, 5C=2/10\nNet: adopt the approved 7.6:1 pair everywhere errors render, or patch text color alone and leave the surface undefined.": "5A #991b1b on #fef2f2 + icon + explicit text (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:23:48.882Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
+476
@@ -0,0 +1,476 @@
|
||||
{
|
||||
"provenance": {
|
||||
"sourceHead": "f26d569e0345cb1131d9ca52d4a43965085c3468",
|
||||
"sourceObservationSha256": "34f0a963e8e5460dea8b2b486155b8bf425582cdad8b0af585e7d4999bf0622b",
|
||||
"publicNativeProofSha256": "a25fa50747cd85aba50fe41be5853d8173701904e948f89070037f5a8bb78972",
|
||||
"historicalOutcome": "plan_ready; missing-quickstart, reversed-arguments and opaque-auth-error seed predicates failed",
|
||||
"paidOutcomesReclassified": false,
|
||||
"scope": "Exact completed retry native questions, options, answers and acknowledgment times. Original failed first and retry outcomes remain unchanged."
|
||||
},
|
||||
"calls": [
|
||||
{
|
||||
"sessionId": "df61df94-90b4-4edc-8a4d-4a354b9d774e",
|
||||
"toolUseId": "toolu_01CbE1po7xEgX3iU2xdNXbDT",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D1 — Add gstack skill routing rules to this project's CLAUDE.md?\nProject/branch/task: gstack-plan-count-ig4gtr on main, starting /plan-devex-review of PLAN.md (EvalKit SDK beta polish).\nELI10: gstack skills work best when the project's CLAUDE.md tells Claude which skill to reach for (bugs → /investigate, ship → /ship, and so on). This is a one-time setup prompt from the skill preamble, unrelated to the review itself. Note: we are in plan mode, so if you pick A the CLAUDE.md edit and commit happen after the review finishes and plan mode exits, not now.\nStakes if we pick wrong: Low. Without routing rules you invoke skills by name manually; with them, Claude auto-routes. Either is reversible with one config command.\nRecommendation: A because auto-routing removes a step every time you ask for a review, fix, or ship.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Add routing rules to CLAUDE.md (recommended)\n ✅ Future requests like \"review this\" or \"ship it\" route to the right gstack skill automatically\n ✅ One short section appended to CLAUDE.md, committed as its own chore commit\n ❌ Cannot be applied during plan mode; it lands after this review completes, and this repo is a review fixture so the benefit here is small\nB) No thanks, I'll invoke skills manually\n ✅ Keeps the fixture CLAUDE.md untouched, no extra commit in this repo\n ✅ Recorded once so this prompt never appears again for this project\n ❌ You type /skill-name yourself every time instead of Claude routing for you\nNet: convenience later versus keeping a fixture repo clean.",
|
||||
"header": "Setup",
|
||||
"options": [
|
||||
{
|
||||
"label": "Add routing rules (recommended)",
|
||||
"description": "Append the gstack routing section to CLAUDE.md and commit it once plan mode exits."
|
||||
},
|
||||
{
|
||||
"label": "No thanks, manual",
|
||||
"description": "Set routing_declined so this prompt never fires again for this project."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D1 — Add gstack skill routing rules to this project's CLAUDE.md?\nProject/branch/task: gstack-plan-count-ig4gtr on main, starting /plan-devex-review of PLAN.md (EvalKit SDK beta polish).\nELI10: gstack skills work best when the project's CLAUDE.md tells Claude which skill to reach for (bugs → /investigate, ship → /ship, and so on). This is a one-time setup prompt from the skill preamble, unrelated to the review itself. Note: we are in plan mode, so if you pick A the CLAUDE.md edit and commit happen after the review finishes and plan mode exits, not now.\nStakes if we pick wrong: Low. Without routing rules you invoke skills by name manually; with them, Claude auto-routes. Either is reversible with one config command.\nRecommendation: A because auto-routing removes a step every time you ask for a review, fix, or ship.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Add routing rules to CLAUDE.md (recommended)\n ✅ Future requests like \"review this\" or \"ship it\" route to the right gstack skill automatically\n ✅ One short section appended to CLAUDE.md, committed as its own chore commit\n ❌ Cannot be applied during plan mode; it lands after this review completes, and this repo is a review fixture so the benefit here is small\nB) No thanks, I'll invoke skills manually\n ✅ Keeps the fixture CLAUDE.md untouched, no extra commit in this repo\n ✅ Recorded once so this prompt never appears again for this project\n ❌ You type /skill-name yourself every time instead of Claude routing for you\nNet: convenience later versus keeping a fixture repo clean.": "Add routing rules (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:13:14.530Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "df61df94-90b4-4edc-8a4d-4a354b9d774e",
|
||||
"toolUseId": "toolu_018RBgqQLz5ttK7QwHaGASMG",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D2 — Let gstack search learnings from your other projects on this machine?\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit SDK beta plan.\nELI10: gstack saves small lessons per project (command quirks, pitfalls). It can also look at lessons saved from your other repos on this machine to spot patterns that apply here. Nothing leaves your machine either way. This is a one-time preference; the review continues right after.\nStakes if we pick wrong: Low and reversible. Too broad means a lesson from an unrelated client codebase might get suggested here. Too narrow means a useful pattern from another repo stays hidden.\nRecommendation: A because this looks like a single-owner machine and cross-project lessons compound fastest for solo developers.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Enable cross-project learnings (recommended)\n ✅ Pitfalls found in one repo surface automatically when the same tool or pattern appears here\n ✅ Stays local; one config flag you can flip back at any time\n ❌ If you work across separate client codebases, a lesson from one could bleed into another's review\nB) Keep learnings project-scoped only\n ✅ Strict isolation between codebases, no cross-contamination of context\n ✅ Still records and reuses lessons within this project\n ❌ A pattern already learned elsewhere has to be rediscovered here from scratch\nNet: faster compounding versus strict per-project isolation.",
|
||||
"header": "Learnings",
|
||||
"options": [
|
||||
{
|
||||
"label": "Enable cross-project (recommended)",
|
||||
"description": "Set cross_project_learnings true; searches stay local to this machine."
|
||||
},
|
||||
{
|
||||
"label": "Project-scoped only",
|
||||
"description": "Set cross_project_learnings false; only this repo's learnings are used."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D2 — Let gstack search learnings from your other projects on this machine?\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit SDK beta plan.\nELI10: gstack saves small lessons per project (command quirks, pitfalls). It can also look at lessons saved from your other repos on this machine to spot patterns that apply here. Nothing leaves your machine either way. This is a one-time preference; the review continues right after.\nStakes if we pick wrong: Low and reversible. Too broad means a lesson from an unrelated client codebase might get suggested here. Too narrow means a useful pattern from another repo stays hidden.\nRecommendation: A because this looks like a single-owner machine and cross-project lessons compound fastest for solo developers.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Enable cross-project learnings (recommended)\n ✅ Pitfalls found in one repo surface automatically when the same tool or pattern appears here\n ✅ Stays local; one config flag you can flip back at any time\n ❌ If you work across separate client codebases, a lesson from one could bleed into another's review\nB) Keep learnings project-scoped only\n ✅ Strict isolation between codebases, no cross-contamination of context\n ✅ Still records and reuses lessons within this project\n ❌ A pattern already learned elsewhere has to be rediscovered here from scratch\nNet: faster compounding versus strict per-project isolation.": "Enable cross-project (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:13:32.616Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "df61df94-90b4-4edc-8a4d-4a354b9d774e",
|
||||
"toolUseId": "toolu_01XNQJDT5JwbCJokeE7VRasH",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D3 — Does this match what your ML engineer experiences today?\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: Before scoring anything I walk the real getting-started path as your developer, using only README.md and the documented contracts. If this walk is wrong, every later recommendation is aimed at the wrong pain. Here is the walk, first person:\n\n\"I'm an ML engineer. I need to score our model's answers before we wire evals into CI. I open the EvalKit README. The first paragraph describes who I am and a 'review posture'; I skip it. Under Getting started I run `python -m pip install evalkit==2.0.0b1`. Twenty seconds, fine. Next line: `python examples/first_eval.py`. I run it: `can't open file 'examples/first_eval.py': No such file or directory`. I look in site-packages. No examples folder. I re-read and spot `python -m evalkit.demo`, run that instead. It prints 'Verifying the sample-project binding with EvalKit CI; inspect https://ci.evalkit.example/checks/...; normally completes within 300s.' Then 'Waiting for CI check: 30s elapsed of 300s.' CI? I have no key, no project, I'm scoring three bundled samples through a mock transport. Five minutes later: example 1: score=0.80, example 2: score=1.00, overall: score=0.90. Six minutes gone; the peer SDK I tried last week took two. Now I try my own data. I create a key, export EVALKIT_API_KEY, and mistype it. `AuthError: request failed`. Is the service down? Is my key wrong? I can't tell.\"\n\nStakes if we pick wrong: If this walk is off, I fix the wrong friction and the beta ships with the real blockers intact.\nRecommendation: A because every step above is taken verbatim from README.md, docs/package-contents.txt, docs/current-contracts.md, and docs/api.md.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Accurate, proceed with this understanding (recommended)\n ✅ Every step traces to a documented contract, so the friction points that follow are evidence-based\n ✅ Lets the review move straight to the five concrete repairs the plan asked for\n ❌ If an undocumented behavior softens any step, the review over-weights it\nB) Some of this is wrong, let me correct it\n ✅ Corrections get folded into the narrative before any scoring happens\n ✅ Cheap to do now; expensive to discover after the plan is written\n ❌ Adds a round trip before the friction-point decisions start\nC) Way off, the actual experience is different\n ✅ Stops a misaimed review before it produces misaimed fixes\n ✅ You describe the real path and I re-trace from your description\n ❌ Discards the documented-contract trace, so I need the real contracts in your reply\nNet: confirm the trace now so the fixes that follow hit real pain, not assumed pain.",
|
||||
"header": "Empathy",
|
||||
"options": [
|
||||
{
|
||||
"label": "Accurate, proceed (recommended)",
|
||||
"description": "The trace matches the documented contracts; use it as the baseline."
|
||||
},
|
||||
{
|
||||
"label": "Partly wrong, correct it",
|
||||
"description": "Tell me which steps differ and I fold the corrections in."
|
||||
},
|
||||
{
|
||||
"label": "Way off",
|
||||
"description": "Describe the actual experience and I re-trace from that."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D3 — Does this match what your ML engineer experiences today?\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: Before scoring anything I walk the real getting-started path as your developer, using only README.md and the documented contracts. If this walk is wrong, every later recommendation is aimed at the wrong pain. Here is the walk, first person:\n\n\"I'm an ML engineer. I need to score our model's answers before we wire evals into CI. I open the EvalKit README. The first paragraph describes who I am and a 'review posture'; I skip it. Under Getting started I run `python -m pip install evalkit==2.0.0b1`. Twenty seconds, fine. Next line: `python examples/first_eval.py`. I run it: `can't open file 'examples/first_eval.py': No such file or directory`. I look in site-packages. No examples folder. I re-read and spot `python -m evalkit.demo`, run that instead. It prints 'Verifying the sample-project binding with EvalKit CI; inspect https://ci.evalkit.example/checks/...; normally completes within 300s.' Then 'Waiting for CI check: 30s elapsed of 300s.' CI? I have no key, no project, I'm scoring three bundled samples through a mock transport. Five minutes later: example 1: score=0.80, example 2: score=1.00, overall: score=0.90. Six minutes gone; the peer SDK I tried last week took two. Now I try my own data. I create a key, export EVALKIT_API_KEY, and mistype it. `AuthError: request failed`. Is the service down? Is my key wrong? I can't tell.\"\n\nStakes if we pick wrong: If this walk is off, I fix the wrong friction and the beta ships with the real blockers intact.\nRecommendation: A because every step above is taken verbatim from README.md, docs/package-contents.txt, docs/current-contracts.md, and docs/api.md.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Accurate, proceed with this understanding (recommended)\n ✅ Every step traces to a documented contract, so the friction points that follow are evidence-based\n ✅ Lets the review move straight to the five concrete repairs the plan asked for\n ❌ If an undocumented behavior softens any step, the review over-weights it\nB) Some of this is wrong, let me correct it\n ✅ Corrections get folded into the narrative before any scoring happens\n ✅ Cheap to do now; expensive to discover after the plan is written\n ❌ Adds a round trip before the friction-point decisions start\nC) Way off, the actual experience is different\n ✅ Stops a misaimed review before it produces misaimed fixes\n ✅ You describe the real path and I re-trace from your description\n ❌ Discards the documented-contract trace, so I need the real contracts in your reply\nNet: confirm the trace now so the fixes that follow hit real pain, not assumed pain.": "Accurate, proceed (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:14:28.433Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "df61df94-90b4-4edc-8a4d-4a354b9d774e",
|
||||
"toolUseId": "toolu_013S28Bj1dxh5fVrdaKBBC95",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D4 — Journey stage INSTALL / HELLO WORLD: the quickstart's first command points at a file that does not ship.\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: README.md line 11 tells the developer to run `python examples/first_eval.py` right after pip install. docs/package-contents.txt says that file is absent from both the published wheel and the release examples archive. So the very first command in the quickstart fails with 'No such file or directory'. The working demo, `python -m evalkit.demo`, is mentioned three lines later but is not the command the developer is told to run first. Your ML engineer hits this at about minute 1, before seeing any value.\nStakes if we pick wrong: A dead first command is the strongest 'this tool is unmaintained' signal a beta can send. It violates 'zero friction at T0' and 'show code in context' (the example the docs promise does not exist).\nRecommendation: A because the demo already works and should be the one golden path; the example file still has a job as the first live run with a key.\nCompleteness: A=10/10, B=7/10, C=6/10, D=1/10\nPros / cons:\nA) Make `python -m evalkit.demo` the quickstart's first command AND ship examples/first_eval.py in the examples archive as the documented first live evaluation (recommended) (human: ~2h / CC: ~10 min)\n ✅ One golden path: install, demo, then key, then first_eval.py with real data, each step producing visible output\n ✅ Closes the package-inventory gap so every file the docs name actually exists where the docs say\n ❌ Requires writing and testing first_eval.py against the live transport before the beta tags\nB) Repoint the quickstart to `python -m evalkit.demo` and delete the examples/first_eval.py reference (human: ~15 min / CC: ~2 min)\n ✅ Zero broken commands in the README with a one-line edit\n ✅ No new artifact to build or test before release\n ❌ Developer is left with no worked example of a live evaluation with their own data; they write it from the API reference\nC) Ship examples/first_eval.py in the package and archive, keep the quickstart order as-is (human: ~2h / CC: ~10 min)\n ✅ The documented command works as written\n ✅ Example file becomes a real, copy-paste artifact\n ❌ Two competing first commands (first_eval.py and evalkit.demo) is the 'choose your own adventure' anti-pattern; and a file run from a package path is awkward after pip install\nD) Acceptable friction, skip\n ✅ No release work\n ✅ Developers who read the whole README find the demo anyway\n ❌ The first thing the beta asks the developer to run fails; that is the minute-1 abandon point\nNet: one working golden path and a real live example, versus a quick edit that leaves the live-run example missing.",
|
||||
"header": "Hello World",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Demo first + ship example (recommended)",
|
||||
"description": "Quickstart runs python -m evalkit.demo first; examples/first_eval.py ships as the first live run."
|
||||
},
|
||||
{
|
||||
"label": "B) Repoint to demo only",
|
||||
"description": "Fix the README command; drop the examples file reference."
|
||||
},
|
||||
{
|
||||
"label": "C) Ship the example, keep order",
|
||||
"description": "Add examples/first_eval.py to package and archive; quickstart unchanged."
|
||||
},
|
||||
{
|
||||
"label": "D) Skip",
|
||||
"description": "Leave the quickstart as-is."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D4 — Journey stage INSTALL / HELLO WORLD: the quickstart's first command points at a file that does not ship.\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: README.md line 11 tells the developer to run `python examples/first_eval.py` right after pip install. docs/package-contents.txt says that file is absent from both the published wheel and the release examples archive. So the very first command in the quickstart fails with 'No such file or directory'. The working demo, `python -m evalkit.demo`, is mentioned three lines later but is not the command the developer is told to run first. Your ML engineer hits this at about minute 1, before seeing any value.\nStakes if we pick wrong: A dead first command is the strongest 'this tool is unmaintained' signal a beta can send. It violates 'zero friction at T0' and 'show code in context' (the example the docs promise does not exist).\nRecommendation: A because the demo already works and should be the one golden path; the example file still has a job as the first live run with a key.\nCompleteness: A=10/10, B=7/10, C=6/10, D=1/10\nPros / cons:\nA) Make `python -m evalkit.demo` the quickstart's first command AND ship examples/first_eval.py in the examples archive as the documented first live evaluation (recommended) (human: ~2h / CC: ~10 min)\n ✅ One golden path: install, demo, then key, then first_eval.py with real data, each step producing visible output\n ✅ Closes the package-inventory gap so every file the docs name actually exists where the docs say\n ❌ Requires writing and testing first_eval.py against the live transport before the beta tags\nB) Repoint the quickstart to `python -m evalkit.demo` and delete the examples/first_eval.py reference (human: ~15 min / CC: ~2 min)\n ✅ Zero broken commands in the README with a one-line edit\n ✅ No new artifact to build or test before release\n ❌ Developer is left with no worked example of a live evaluation with their own data; they write it from the API reference\nC) Ship examples/first_eval.py in the package and archive, keep the quickstart order as-is (human: ~2h / CC: ~10 min)\n ✅ The documented command works as written\n ✅ Example file becomes a real, copy-paste artifact\n ❌ Two competing first commands (first_eval.py and evalkit.demo) is the 'choose your own adventure' anti-pattern; and a file run from a package path is awkward after pip install\nD) Acceptable friction, skip\n ✅ No release work\n ✅ Developers who read the whole README find the demo anyway\n ❌ The first thing the beta asks the developer to run fails; that is the minute-1 abandon point\nNet: one working golden path and a real live example, versus a quick edit that leaves the live-run example missing.": "A) Demo first + ship example (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:15:00.631Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "df61df94-90b4-4edc-8a4d-4a354b9d774e",
|
||||
"toolUseId": "toolu_01WHk4S8opK1mfQjfAjMfAJm",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D5 — Journey stage HELLO WORLD: the keyless demo blocks on a 5-minute remote CI check, which alone exceeds the settled 2-minute target.\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: docs/current-contracts.md says every first local evaluation, including `python -m evalkit.demo`, must wait for a successful remote CI check that normally takes 300 seconds, with no skip and no offline path. The demo scores bundled JSON through a mock transport; the CI check verifies a sample-project binding the developer never chose. docs/benchmarks.md measured EvalKit at 6 minutes, 5 of them this wait, against peers at 2 to 4 minutes and an agreed target under 2 minutes. If CI is degraded the demo fails outright with EVALKIT_CI_TIMEOUT, and the developer has no way to see any score. The progress lines and check URL are good, but they make the wait legible rather than short.\nStakes if we pick wrong: The under-2-minute target is arithmetically unreachable while the gate stays on the demo path. Your ML engineer waits five minutes to score three canned samples, or hits a CI outage and sees nothing. This is the single largest 'zero friction at T0' violation in the plan.\nRecommendation: A because the demo path has nothing to verify remotely, and the first live run should return its result while the binding check reports alongside, not in front of it.\nCompleteness: A=10/10, B=7/10, C=5/10, D=1/10\nPros / cons:\nA) Remove the CI check from the demo/mock-transport path; on the first live evaluation run the binding check non-blocking (result returns immediately, check status prints when it completes, failure surfaces as a warning with the check URL) (recommended) (human: ~3 days / CC: ~45 min)\n ✅ Demo TTHW drops from ~6 min to under 1 min; the settled Champion target becomes reachable\n ✅ Keeps the binding verification and its good messaging for live runs, without ever putting it in front of a result\n ❌ Changes a documented contract; the first live result can land before the binding is confirmed, so the warning path must be unmissable\nB) Exempt only the demo/mock-transport path; keep the blocking 5-minute check on the first live evaluation (human: ~1 day / CC: ~15 min)\n ✅ Smallest change that makes the demo fast and keyless in fact, not just in name\n ✅ Live-run contract stays exactly as documented today\n ❌ First live run with real data still waits up to 5 min, so the measured 'first real evaluation result' stays near 6 min\nC) Keep the gate by default; add an explicit escape hatch (`EVALKIT_SKIP_CI_CHECK=1` env var and `python -m evalkit.demo --offline`) (human: ~1 day / CC: ~15 min)\n ✅ Power users and CI runners can bypass the wait when they know what they are doing\n ✅ No change to the default documented behavior\n ❌ Default TTHW stays at 6 min; a flag the developer must discover is not a fix for a first-run wait (violates 'decide for me')\nD) Keep the mandatory check as documented\n ✅ No runtime change before beta\n ✅ Progress lines and timeout messaging already make the wait understandable\n ❌ Under-2-minute target cannot be met; a CI outage means the keyless demo cannot produce any score\nNet: remove the wait from the paths that do not need it and make the remaining check informative rather than blocking, versus preserving a contract that guarantees the benchmark is missed.",
|
||||
"header": "CI gate",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) No gate on demo; non-blocking on live (recommended)",
|
||||
"description": "Demo returns immediately; first live run returns result while the binding check reports alongside."
|
||||
},
|
||||
{
|
||||
"label": "B) Exempt demo only",
|
||||
"description": "Demo skips the check; first live evaluation still blocks up to 5 min."
|
||||
},
|
||||
{
|
||||
"label": "C) Keep gate, add skip flag",
|
||||
"description": "Default unchanged; add env var and --offline escape hatch."
|
||||
},
|
||||
{
|
||||
"label": "D) Keep as documented",
|
||||
"description": "Retain the mandatory blocking check on all first runs."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D5 — Journey stage HELLO WORLD: the keyless demo blocks on a 5-minute remote CI check, which alone exceeds the settled 2-minute target.\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: docs/current-contracts.md says every first local evaluation, including `python -m evalkit.demo`, must wait for a successful remote CI check that normally takes 300 seconds, with no skip and no offline path. The demo scores bundled JSON through a mock transport; the CI check verifies a sample-project binding the developer never chose. docs/benchmarks.md measured EvalKit at 6 minutes, 5 of them this wait, against peers at 2 to 4 minutes and an agreed target under 2 minutes. If CI is degraded the demo fails outright with EVALKIT_CI_TIMEOUT, and the developer has no way to see any score. The progress lines and check URL are good, but they make the wait legible rather than short.\nStakes if we pick wrong: The under-2-minute target is arithmetically unreachable while the gate stays on the demo path. Your ML engineer waits five minutes to score three canned samples, or hits a CI outage and sees nothing. This is the single largest 'zero friction at T0' violation in the plan.\nRecommendation: A because the demo path has nothing to verify remotely, and the first live run should return its result while the binding check reports alongside, not in front of it.\nCompleteness: A=10/10, B=7/10, C=5/10, D=1/10\nPros / cons:\nA) Remove the CI check from the demo/mock-transport path; on the first live evaluation run the binding check non-blocking (result returns immediately, check status prints when it completes, failure surfaces as a warning with the check URL) (recommended) (human: ~3 days / CC: ~45 min)\n ✅ Demo TTHW drops from ~6 min to under 1 min; the settled Champion target becomes reachable\n ✅ Keeps the binding verification and its good messaging for live runs, without ever putting it in front of a result\n ❌ Changes a documented contract; the first live result can land before the binding is confirmed, so the warning path must be unmissable\nB) Exempt only the demo/mock-transport path; keep the blocking 5-minute check on the first live evaluation (human: ~1 day / CC: ~15 min)\n ✅ Smallest change that makes the demo fast and keyless in fact, not just in name\n ✅ Live-run contract stays exactly as documented today\n ❌ First live run with real data still waits up to 5 min, so the measured 'first real evaluation result' stays near 6 min\nC) Keep the gate by default; add an explicit escape hatch (`EVALKIT_SKIP_CI_CHECK=1` env var and `python -m evalkit.demo --offline`) (human: ~1 day / CC: ~15 min)\n ✅ Power users and CI runners can bypass the wait when they know what they are doing\n ✅ No change to the default documented behavior\n ❌ Default TTHW stays at 6 min; a flag the developer must discover is not a fix for a first-run wait (violates 'decide for me')\nD) Keep the mandatory check as documented\n ✅ No runtime change before beta\n ✅ Progress lines and timeout messaging already make the wait understandable\n ❌ Under-2-minute target cannot be met; a CI outage means the keyless demo cannot produce any score\nNet: remove the wait from the paths that do not need it and make the remaining check informative rather than blocking, versus preserving a contract that guarantees the benchmark is missed.": "A) No gate on demo; non-blocking on live (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:15:34.831Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "df61df94-90b4-4edc-8a4d-4a354b9d774e",
|
||||
"toolUseId": "toolu_0192184CCwGGfQWFKERXwCDH",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D6 — Journey stage REAL USAGE: `run_eval(dataset, evaluator)` and `run_batch(evaluator, dataset)` take the same two concepts in opposite positional order.\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: docs/api.md says both evaluation functions accept positional arguments for the same two things, but in reversed order, and calls this intentional. Neither requires keywords. Your ML engineer learns `run_eval(dataset, evaluator)` from the demo, then writes `run_batch(dataset, evaluator)` for their real job by pattern-matching, and passes the evaluator where the dataset goes. Depending on the types involved this is either a confusing TypeError deep in the call, or a silent wrong result. The API fails the 'use it correctly after seeing one example' test.\nStakes if we pick wrong: This is a beta; the positional order becomes a contract the moment it ships. Fixing it later is the v1-to-v2 breaking-change problem all over again. It violates 'pit of success' and the Usable characteristic (consistency).\nRecommendation: A because 2.0 is already the breaking release, and a swap detector turns the one remaining footgun into a helpful message instead of a wrong answer.\nCompleteness: A=10/10, B=7/10, C=4/10, D=1/10\nPros / cons:\nA) Align both to `(dataset, evaluator)`, accept keywords, and in `run_batch` detect a swapped call by argument type and raise `EVALKIT_ARGS_SWAPPED` with the corrected call in the message (recommended) (human: ~1 day / CC: ~20 min)\n ✅ One order to remember across the whole surface; the demo example teaches the real API\n ✅ Anyone carrying the old `run_batch` order gets told exactly what to type, not a TypeError from inside the library\n ❌ Adds a small type-inspection branch that must be tested against both argument types and custom evaluator subclasses\nB) Align both to `(dataset, evaluator)` and document the `run_batch` change in the 2.0 changelog with no runtime detection (human: ~2h / CC: ~5 min)\n ✅ Consistent API with minimal code change\n ✅ 2.0 is a major version, so a documented positional change is legitimate\n ❌ A 1.x caller's swapped `run_batch` fails with whatever error the wrong types produce, with no pointer to the fix\nC) Keep both orders; add a bold note in docs/api.md and the docstrings (human: ~30 min / CC: ~3 min)\n ✅ No runtime or signature change before beta\n ✅ Docstring warning shows up in editor hover\n ❌ Developers copy the pattern from one call to the other without reading the note; the footgun ships as a permanent contract\nD) Keep as documented, skip\n ✅ Zero work\n ✅ Existing 1.x `run_batch` callers keep their exact order\n ❌ Locks an inconsistent public surface into the 2.x line\nNet: consistent order plus a helpful swap error now, versus documenting around an inconsistency the beta would freeze.",
|
||||
"header": "Signatures",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Align + swap detector (recommended)",
|
||||
"description": "Both take (dataset, evaluator); run_batch raises EVALKIT_ARGS_SWAPPED with the corrected call."
|
||||
},
|
||||
{
|
||||
"label": "B) Align, changelog only",
|
||||
"description": "Both take (dataset, evaluator); no runtime detection."
|
||||
},
|
||||
{
|
||||
"label": "C) Keep orders, document",
|
||||
"description": "Add a warning to docs/api.md and docstrings."
|
||||
},
|
||||
{
|
||||
"label": "D) Skip",
|
||||
"description": "Ship the reversed positional order as documented."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D6 — Journey stage REAL USAGE: `run_eval(dataset, evaluator)` and `run_batch(evaluator, dataset)` take the same two concepts in opposite positional order.\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: docs/api.md says both evaluation functions accept positional arguments for the same two things, but in reversed order, and calls this intentional. Neither requires keywords. Your ML engineer learns `run_eval(dataset, evaluator)` from the demo, then writes `run_batch(dataset, evaluator)` for their real job by pattern-matching, and passes the evaluator where the dataset goes. Depending on the types involved this is either a confusing TypeError deep in the call, or a silent wrong result. The API fails the 'use it correctly after seeing one example' test.\nStakes if we pick wrong: This is a beta; the positional order becomes a contract the moment it ships. Fixing it later is the v1-to-v2 breaking-change problem all over again. It violates 'pit of success' and the Usable characteristic (consistency).\nRecommendation: A because 2.0 is already the breaking release, and a swap detector turns the one remaining footgun into a helpful message instead of a wrong answer.\nCompleteness: A=10/10, B=7/10, C=4/10, D=1/10\nPros / cons:\nA) Align both to `(dataset, evaluator)`, accept keywords, and in `run_batch` detect a swapped call by argument type and raise `EVALKIT_ARGS_SWAPPED` with the corrected call in the message (recommended) (human: ~1 day / CC: ~20 min)\n ✅ One order to remember across the whole surface; the demo example teaches the real API\n ✅ Anyone carrying the old `run_batch` order gets told exactly what to type, not a TypeError from inside the library\n ❌ Adds a small type-inspection branch that must be tested against both argument types and custom evaluator subclasses\nB) Align both to `(dataset, evaluator)` and document the `run_batch` change in the 2.0 changelog with no runtime detection (human: ~2h / CC: ~5 min)\n ✅ Consistent API with minimal code change\n ✅ 2.0 is a major version, so a documented positional change is legitimate\n ❌ A 1.x caller's swapped `run_batch` fails with whatever error the wrong types produce, with no pointer to the fix\nC) Keep both orders; add a bold note in docs/api.md and the docstrings (human: ~30 min / CC: ~3 min)\n ✅ No runtime or signature change before beta\n ✅ Docstring warning shows up in editor hover\n ❌ Developers copy the pattern from one call to the other without reading the note; the footgun ships as a permanent contract\nD) Keep as documented, skip\n ✅ Zero work\n ✅ Existing 1.x `run_batch` callers keep their exact order\n ❌ Locks an inconsistent public surface into the 2.x line\nNet: consistent order plus a helpful swap error now, versus documenting around an inconsistency the beta would freeze.": "A) Align + swap detector (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:16:03.003Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "df61df94-90b4-4edc-8a4d-4a354b9d774e",
|
||||
"toolUseId": "toolu_01UmtXf9KxaaYLR51odEMjqK",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D7 — Journey stage DEBUG: an invalid API key raises `AuthError(\"request failed\")` with no code, no cause, and no fix.\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: docs/api.md says a bad key produces exactly `AuthError(\"request failed\")`. docs/current-contracts.md says every other error in the SDK already names the cause, the argument or file involved, and an actionable fix. Auth is the one exception, and it is the first error your ML engineer is likely to hit: they just copied a key once from the console, exported it, and typed it wrong or exported it in a different shell. 'request failed' does not tell them whether the key is wrong, expired, revoked, scoped to another project, or whether the service is down. They leave the terminal to go check the console and the status page.\nStakes if we pick wrong: This is the moment the developer moves from the free demo to a live run with their own data. An opaque failure here reads as 'the service is flaky', not 'I mistyped'. It violates 'fight uncertainty' (problem + cause + fix) and is the only error in the SDK below the bar the rest already meets.\nRecommendation: A because the rest of the SDK's errors already follow this formula; auth should match it, and the console URL already exists in the README.\nCompleteness: A=10/10, B=7/10, C=1/10\nPros / cons:\nA) Raise `AuthError` with code `EVALKIT_AUTH_INVALID_KEY`, a message stating the key from `EVALKIT_API_KEY` was rejected (redacted to prefix + last 4), the likely causes (typo, revoked, wrong project), the fix (create or rotate at https://console.evalkit.example/settings/api-keys and re-export), and a docs link; distinguish a missing key (`EVALKIT_AUTH_MISSING_KEY`) from a rejected one (recommended) (human: ~1 day / CC: ~15 min)\n ✅ Developer fixes the key without leaving the terminal; matches the problem + cause + fix formula every other SDK error already uses\n ✅ Separate missing-vs-rejected codes remove the most common false alarm (forgot to export in this shell)\n ❌ Needs the server to return a distinguishable auth status so the client can tell rejected from unreachable; verify that before promising the message\nB) Improve the message text only: `AuthError(\"API key rejected. Check EVALKIT_API_KEY and create or rotate a key at <console URL>.\")` with no error code and no missing-vs-rejected split (human: ~1h / CC: ~5 min)\n ✅ Immediately actionable with a one-string change\n ✅ No new error classes or server dependency\n ❌ No code to grep or link to, and 'forgot to export' still looks identical to 'key revoked'\nC) Keep `AuthError(\"request failed\")` as documented\n ✅ No change before beta\n ✅ Existing tests asserting the message keep passing\n ❌ The first live-run failure your persona hits gives them nothing to act on\nNet: bring the one substandard error up to the SDK's own bar, versus shipping the beta with its most likely first failure unexplained.",
|
||||
"header": "Auth error",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Coded, causal, fix + link (recommended)",
|
||||
"description": "EVALKIT_AUTH_INVALID_KEY / EVALKIT_AUTH_MISSING_KEY with cause, redacted key, console URL, docs link."
|
||||
},
|
||||
{
|
||||
"label": "B) Better message only",
|
||||
"description": "Actionable text, no code, no missing-vs-rejected split."
|
||||
},
|
||||
{
|
||||
"label": "C) Keep as documented",
|
||||
"description": "Ship AuthError(\"request failed\")."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D7 — Journey stage DEBUG: an invalid API key raises `AuthError(\"request failed\")` with no code, no cause, and no fix.\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: docs/api.md says a bad key produces exactly `AuthError(\"request failed\")`. docs/current-contracts.md says every other error in the SDK already names the cause, the argument or file involved, and an actionable fix. Auth is the one exception, and it is the first error your ML engineer is likely to hit: they just copied a key once from the console, exported it, and typed it wrong or exported it in a different shell. 'request failed' does not tell them whether the key is wrong, expired, revoked, scoped to another project, or whether the service is down. They leave the terminal to go check the console and the status page.\nStakes if we pick wrong: This is the moment the developer moves from the free demo to a live run with their own data. An opaque failure here reads as 'the service is flaky', not 'I mistyped'. It violates 'fight uncertainty' (problem + cause + fix) and is the only error in the SDK below the bar the rest already meets.\nRecommendation: A because the rest of the SDK's errors already follow this formula; auth should match it, and the console URL already exists in the README.\nCompleteness: A=10/10, B=7/10, C=1/10\nPros / cons:\nA) Raise `AuthError` with code `EVALKIT_AUTH_INVALID_KEY`, a message stating the key from `EVALKIT_API_KEY` was rejected (redacted to prefix + last 4), the likely causes (typo, revoked, wrong project), the fix (create or rotate at https://console.evalkit.example/settings/api-keys and re-export), and a docs link; distinguish a missing key (`EVALKIT_AUTH_MISSING_KEY`) from a rejected one (recommended) (human: ~1 day / CC: ~15 min)\n ✅ Developer fixes the key without leaving the terminal; matches the problem + cause + fix formula every other SDK error already uses\n ✅ Separate missing-vs-rejected codes remove the most common false alarm (forgot to export in this shell)\n ❌ Needs the server to return a distinguishable auth status so the client can tell rejected from unreachable; verify that before promising the message\nB) Improve the message text only: `AuthError(\"API key rejected. Check EVALKIT_API_KEY and create or rotate a key at <console URL>.\")` with no error code and no missing-vs-rejected split (human: ~1h / CC: ~5 min)\n ✅ Immediately actionable with a one-string change\n ✅ No new error classes or server dependency\n ❌ No code to grep or link to, and 'forgot to export' still looks identical to 'key revoked'\nC) Keep `AuthError(\"request failed\")` as documented\n ✅ No change before beta\n ✅ Existing tests asserting the message keep passing\n ❌ The first live-run failure your persona hits gives them nothing to act on\nNet: bring the one substandard error up to the SDK's own bar, versus shipping the beta with its most likely first failure unexplained.": "A) Coded, causal, fix + link (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:16:31.185Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "df61df94-90b4-4edc-8a4d-4a354b9d774e",
|
||||
"toolUseId": "toolu_01SWnVeRYZ978kcGVuf1GcqR",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D8 — Journey stage UPGRADE: v2 removes `Client.evaluate()` immediately in favor of `Client.run()`, with no alias, warning, guide, or codemod.\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: docs/api.md says version 2 renames the client's main method and deletes the old name on the spot. A team on 1.x that bumps to 2.0.0b1 gets `AttributeError: 'Client' object has no attribute 'evaluate'` at the first call site, with nothing pointing at `run()`. The changelog is otherwise complete, so this is the one breaking change without a landing pad. Your ML engineer is exactly the person who wired 1.x into CI and will see this as a red pipeline, not a rename.\nStakes if we pick wrong: Upgrades should be boring. One unannounced removal teaches every 1.x user that minor-looking bumps break production, and they pin forever. This is the Credible characteristic and the 'upgrade fear' pattern directly.\nRecommendation: A because a one-release alias with a warning costs almost nothing and turns a red pipeline into a one-line diff the developer sees coming.\nCompleteness: A=10/10, B=7/10, C=5/10, D=1/10\nPros / cons:\nA) Keep `Client.evaluate()` as a thin alias for `run()` through the 2.x line, emit `DeprecationWarning: Client.evaluate() is deprecated, use Client.run(); removal in 3.0`, add a 'Migrating from 1.x' section to the changelog and docs, and ship a one-line codemod (`python -m evalkit.migrate --from 1`) that rewrites the call sites (recommended) (human: ~2 days / CC: ~30 min)\n ✅ 1.x code keeps working on upgrade day; the warning names the exact replacement and the removal version\n ✅ Codemod plus migration guide means the rename is a single command, not a grep-and-hope\n ❌ Two public names for one method for the 2.x lifetime, and the codemod needs tests against real 1.x call patterns\nB) Alias + `DeprecationWarning` + migration guide, no codemod (human: ~half day / CC: ~10 min)\n ✅ Nothing breaks on upgrade; developers get a clear pointer and a written guide\n ✅ No migration tool to build or maintain\n ❌ The rename is still manual across every call site; teams with many call sites defer the upgrade\nC) No alias; raise a targeted error instead of `AttributeError`: `Client.evaluate() was renamed to Client.run() in 2.0, see <migration URL>` (human: ~2h / CC: ~5 min)\n ✅ Clean 2.0 surface with one method name\n ✅ The failure at least tells the developer what happened and what to type\n ❌ Still a hard break on upgrade day; CI goes red before anyone reads the message\nD) Remove immediately as documented\n ✅ Simplest 2.0 codebase\n ✅ Changelog already lists the change\n ❌ Bare `AttributeError` in production with no pointer to the fix; teaches users to pin and never upgrade\nNet: a boring upgrade with alias, warning, guide, and codemod, versus a clean surface bought with a red pipeline for every 1.x user.",
|
||||
"header": "Upgrade",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Alias + warning + guide + codemod (recommended)",
|
||||
"description": "evaluate() stays as deprecated alias through 2.x; migration guide; python -m evalkit.migrate."
|
||||
},
|
||||
{
|
||||
"label": "B) Alias + warning + guide",
|
||||
"description": "Same landing pad without the codemod."
|
||||
},
|
||||
{
|
||||
"label": "C) Targeted rename error",
|
||||
"description": "No alias; a clear error names run() and links the guide."
|
||||
},
|
||||
{
|
||||
"label": "D) Remove as documented",
|
||||
"description": "Ship the immediate removal."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D8 — Journey stage UPGRADE: v2 removes `Client.evaluate()` immediately in favor of `Client.run()`, with no alias, warning, guide, or codemod.\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: docs/api.md says version 2 renames the client's main method and deletes the old name on the spot. A team on 1.x that bumps to 2.0.0b1 gets `AttributeError: 'Client' object has no attribute 'evaluate'` at the first call site, with nothing pointing at `run()`. The changelog is otherwise complete, so this is the one breaking change without a landing pad. Your ML engineer is exactly the person who wired 1.x into CI and will see this as a red pipeline, not a rename.\nStakes if we pick wrong: Upgrades should be boring. One unannounced removal teaches every 1.x user that minor-looking bumps break production, and they pin forever. This is the Credible characteristic and the 'upgrade fear' pattern directly.\nRecommendation: A because a one-release alias with a warning costs almost nothing and turns a red pipeline into a one-line diff the developer sees coming.\nCompleteness: A=10/10, B=7/10, C=5/10, D=1/10\nPros / cons:\nA) Keep `Client.evaluate()` as a thin alias for `run()` through the 2.x line, emit `DeprecationWarning: Client.evaluate() is deprecated, use Client.run(); removal in 3.0`, add a 'Migrating from 1.x' section to the changelog and docs, and ship a one-line codemod (`python -m evalkit.migrate --from 1`) that rewrites the call sites (recommended) (human: ~2 days / CC: ~30 min)\n ✅ 1.x code keeps working on upgrade day; the warning names the exact replacement and the removal version\n ✅ Codemod plus migration guide means the rename is a single command, not a grep-and-hope\n ❌ Two public names for one method for the 2.x lifetime, and the codemod needs tests against real 1.x call patterns\nB) Alias + `DeprecationWarning` + migration guide, no codemod (human: ~half day / CC: ~10 min)\n ✅ Nothing breaks on upgrade; developers get a clear pointer and a written guide\n ✅ No migration tool to build or maintain\n ❌ The rename is still manual across every call site; teams with many call sites defer the upgrade\nC) No alias; raise a targeted error instead of `AttributeError`: `Client.evaluate() was renamed to Client.run() in 2.0, see <migration URL>` (human: ~2h / CC: ~5 min)\n ✅ Clean 2.0 surface with one method name\n ✅ The failure at least tells the developer what happened and what to type\n ❌ Still a hard break on upgrade day; CI goes red before anyone reads the message\nD) Remove immediately as documented\n ✅ Simplest 2.0 codebase\n ✅ Changelog already lists the change\n ❌ Bare `AttributeError` in production with no pointer to the fix; teaches users to pin and never upgrade\nNet: a boring upgrade with alias, warning, guide, and codemod, versus a clean surface bought with a red pipeline for every 1.x user.": "A) Alias + warning + guide + codemod (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:16:57.330Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "df61df94-90b4-4edc-8a4d-4a354b9d774e",
|
||||
"toolUseId": "toolu_015zCsLgV4jQUmM74dL6wbU3",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D9 — Journey stage DISCOVER: the README opens with internal planning language instead of what EvalKit does and how to run it.\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: README.md ships inside the wheel (docs/package-contents.txt) and is the PyPI landing page. Its first paragraph currently tells the reader who 'the primary developer' is and that 'the agreed review posture is DX POLISH'. Lines 39-40 say the runtime 'is maintained separately from this release-planning repo'. That is text for the release team, not for an ML engineer deciding in 10 seconds whether to pip install. The install and demo commands, which are the actual hook, sit below it. This is minor next to D4 through D8, but DX POLISH means every touchpoint, and the README is the first one.\nStakes if we pick wrong: Low but real. A landing page that reads like an internal memo costs a few seconds of trust at the moment the developer is most likely to bounce. It touches 'zero friction at T0' and the Findable characteristic.\nRecommendation: A because the README is public surface area and the planning notes already have a home in PLAN.md and docs/.\nCompleteness: A=10/10, B=6/10, C=2/10\nPros / cons:\nA) Rewrite the README opening: one sentence on what EvalKit does, then install, then `python -m evalkit.demo` with its expected output, all above the fold; move persona, review-posture, and 'maintained separately' text into PLAN.md or docs/ (recommended) (human: ~1h / CC: ~5 min)\n ✅ A developer sees value prop, install, and a working command in the first screen, matching the settled demo vehicle\n ✅ Internal planning context is preserved where the release team looks for it, not on PyPI\n ❌ Touches README structure, so the D4 quickstart edit and this edit should land together to avoid two rewrites\nB) Delete only the review-posture and 'maintained separately' sentences; leave the rest of the intro as-is (human: ~10 min / CC: ~1 min)\n ✅ Removes the two sentences most obviously not meant for developers\n ✅ Minimal diff, no restructuring\n ❌ Persona description still leads; install and demo command still sit below a paragraph of framing\nC) Acceptable friction, skip\n ✅ No README change beyond D4\n ✅ Developers who scroll find the commands\n ❌ PyPI landing page keeps reading as an internal planning note\nNet: a landing page built around the demo command, versus leaving planning prose on the public front door.",
|
||||
"header": "Discover",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Rewrite opening around the demo (recommended)",
|
||||
"description": "Value prop, install, demo command and output first; move planning text to PLAN.md/docs."
|
||||
},
|
||||
{
|
||||
"label": "B) Trim two sentences",
|
||||
"description": "Remove review-posture and maintained-separately lines only."
|
||||
},
|
||||
{
|
||||
"label": "C) Skip",
|
||||
"description": "Leave the README opening as-is."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D9 — Journey stage DISCOVER: the README opens with internal planning language instead of what EvalKit does and how to run it.\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: README.md ships inside the wheel (docs/package-contents.txt) and is the PyPI landing page. Its first paragraph currently tells the reader who 'the primary developer' is and that 'the agreed review posture is DX POLISH'. Lines 39-40 say the runtime 'is maintained separately from this release-planning repo'. That is text for the release team, not for an ML engineer deciding in 10 seconds whether to pip install. The install and demo commands, which are the actual hook, sit below it. This is minor next to D4 through D8, but DX POLISH means every touchpoint, and the README is the first one.\nStakes if we pick wrong: Low but real. A landing page that reads like an internal memo costs a few seconds of trust at the moment the developer is most likely to bounce. It touches 'zero friction at T0' and the Findable characteristic.\nRecommendation: A because the README is public surface area and the planning notes already have a home in PLAN.md and docs/.\nCompleteness: A=10/10, B=6/10, C=2/10\nPros / cons:\nA) Rewrite the README opening: one sentence on what EvalKit does, then install, then `python -m evalkit.demo` with its expected output, all above the fold; move persona, review-posture, and 'maintained separately' text into PLAN.md or docs/ (recommended) (human: ~1h / CC: ~5 min)\n ✅ A developer sees value prop, install, and a working command in the first screen, matching the settled demo vehicle\n ✅ Internal planning context is preserved where the release team looks for it, not on PyPI\n ❌ Touches README structure, so the D4 quickstart edit and this edit should land together to avoid two rewrites\nB) Delete only the review-posture and 'maintained separately' sentences; leave the rest of the intro as-is (human: ~10 min / CC: ~1 min)\n ✅ Removes the two sentences most obviously not meant for developers\n ✅ Minimal diff, no restructuring\n ❌ Persona description still leads; install and demo command still sit below a paragraph of framing\nC) Acceptable friction, skip\n ✅ No README change beyond D4\n ✅ Developers who scroll find the commands\n ❌ PyPI landing page keeps reading as an internal planning note\nNet: a landing page built around the demo command, versus leaving planning prose on the public front door.": "A) Rewrite opening around the demo (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:17:47.149Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "df61df94-90b4-4edc-8a4d-4a354b9d774e",
|
||||
"toolUseId": "toolu_01T9AueotiH5G3iNM8wNKBPz",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D10 — First-time developer roleplay: which confusion points should the plan address?\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: I replayed the getting-started flow as your ML engineer with a clock running, using only what ships today. Each numbered line is a moment of confusion grounded in a specific file. D4 through D9 already decided fixes for most of them; this question confirms the set and catches the two not yet covered (#5 and #6).\n\nFIRST-TIME DEVELOPER REPORT\nPersona: ML engineer, Python daily, terminal-first\nAttempting: EvalKit 2.0.0b1 getting started\n\nT+0:00 Open README. Intro is about 'the primary developer' and a 'review posture'. Skim past. (#1, README.md:3-6, fixed by D9)\nT+0:20 pip install evalkit==2.0.0b1. Works.\nT+0:45 Run `python examples/first_eval.py`. 'No such file or directory'. Check site-packages, no examples/. (#2, README.md:11 vs docs/package-contents.txt, fixed by D4)\nT+1:30 Re-read, find `python -m evalkit.demo`. Run it. Output: 'Verifying the sample-project binding with EvalKit CI... normally completes within 300s.' Why is a keyless mock demo calling CI? (#3, docs/current-contracts.md:3-5, fixed by D5)\nT+2:00 'Waiting for CI check: 30s elapsed of 300s.' Already past the peer SDK's total time. Consider Ctrl-C. (#3)\nT+6:30 Three score lines print. Format is fine. No 'next step' hint after the output; I go back to the README to find out how to run my own data. (#5, README.md:31-36: demo output ends without pointing at the key step or first_eval.py; NOT yet covered)\nT+8:00 Create key in console, export EVALKIT_API_KEY, mistype it. `AuthError: request failed`. Check status page. (#4, docs/api.md:11-13, fixed by D7)\nT+9:00 Write run_batch(dataset, evaluator) by copying run_eval's shape. Wrong order. (#6, docs/api.md:3-9, fixed by D6)\nT+9:30 Also: the README says 'copy the value once', but nothing says which shell or how to persist it; I export in one terminal and run in another. (#7, README.md:25-29; partially covered by D7's EVALKIT_AUTH_MISSING_KEY message; a one-line note in the README would close it)\n\nStakes if we pick wrong: Skipping #5 leaves the demo as a dead end after its best moment; skipping #7 leaves the most common self-inflicted auth failure to the error message alone.\nRecommendation: A because #1-#4 and #6 are already decided and #5 and #7 are each a few lines of README or demo output.\nCompleteness: A=10/10, B=n/a (depends on picks), C=8/10, D=3/10\nPros / cons:\nA) All of them: also add a 'Next: get a key and run examples/first_eval.py' line to the demo output (#5) and a shell-persistence note next to the export instruction (#7) (recommended) (human: ~1h / CC: ~5 min)\n ✅ The demo hands the developer directly to the live step instead of back to the README\n ✅ Closes the 'exported in the wrong shell' trap before it becomes an auth error\n ❌ Two more small edits to README and demo.py to coordinate with D4 and D9\nB) Let me pick which ones matter\n ✅ You choose exactly which of #5 and #7 join the plan\n ✅ Nothing lands you did not explicitly approve\n ❌ One more round trip for two small items\nC) Only the ones already decided (#1-#4, #6); skip #5 and #7\n ✅ No further edits beyond D4-D9\n ✅ The big blockers are all covered\n ❌ Demo output still ends without a next step; wrong-shell export still relies on the error message\nD) Unrealistic, our developers already know the flow\n ✅ No extra work\n ✅ Valid if the beta only goes to people who used 1.x\n ❌ The internal study measured a real developer at 6 minutes on this exact path\nNet: close the two small gaps now while README and demo.py are already open, or leave them for the post-beta survey to find.",
|
||||
"header": "Roleplay",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) All, incl. #5 and #7 (recommended)",
|
||||
"description": "Add a next-step line to demo output and a shell-persistence note by the export instruction."
|
||||
},
|
||||
{
|
||||
"label": "B) Let me pick",
|
||||
"description": "You choose which of #5 and #7 to include."
|
||||
},
|
||||
{
|
||||
"label": "C) Only the decided ones",
|
||||
"description": "Skip #5 and #7."
|
||||
},
|
||||
{
|
||||
"label": "D) Unrealistic",
|
||||
"description": "Our developers already know this flow."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D10 — First-time developer roleplay: which confusion points should the plan address?\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: I replayed the getting-started flow as your ML engineer with a clock running, using only what ships today. Each numbered line is a moment of confusion grounded in a specific file. D4 through D9 already decided fixes for most of them; this question confirms the set and catches the two not yet covered (#5 and #6).\n\nFIRST-TIME DEVELOPER REPORT\nPersona: ML engineer, Python daily, terminal-first\nAttempting: EvalKit 2.0.0b1 getting started\n\nT+0:00 Open README. Intro is about 'the primary developer' and a 'review posture'. Skim past. (#1, README.md:3-6, fixed by D9)\nT+0:20 pip install evalkit==2.0.0b1. Works.\nT+0:45 Run `python examples/first_eval.py`. 'No such file or directory'. Check site-packages, no examples/. (#2, README.md:11 vs docs/package-contents.txt, fixed by D4)\nT+1:30 Re-read, find `python -m evalkit.demo`. Run it. Output: 'Verifying the sample-project binding with EvalKit CI... normally completes within 300s.' Why is a keyless mock demo calling CI? (#3, docs/current-contracts.md:3-5, fixed by D5)\nT+2:00 'Waiting for CI check: 30s elapsed of 300s.' Already past the peer SDK's total time. Consider Ctrl-C. (#3)\nT+6:30 Three score lines print. Format is fine. No 'next step' hint after the output; I go back to the README to find out how to run my own data. (#5, README.md:31-36: demo output ends without pointing at the key step or first_eval.py; NOT yet covered)\nT+8:00 Create key in console, export EVALKIT_API_KEY, mistype it. `AuthError: request failed`. Check status page. (#4, docs/api.md:11-13, fixed by D7)\nT+9:00 Write run_batch(dataset, evaluator) by copying run_eval's shape. Wrong order. (#6, docs/api.md:3-9, fixed by D6)\nT+9:30 Also: the README says 'copy the value once', but nothing says which shell or how to persist it; I export in one terminal and run in another. (#7, README.md:25-29; partially covered by D7's EVALKIT_AUTH_MISSING_KEY message; a one-line note in the README would close it)\n\nStakes if we pick wrong: Skipping #5 leaves the demo as a dead end after its best moment; skipping #7 leaves the most common self-inflicted auth failure to the error message alone.\nRecommendation: A because #1-#4 and #6 are already decided and #5 and #7 are each a few lines of README or demo output.\nCompleteness: A=10/10, B=n/a (depends on picks), C=8/10, D=3/10\nPros / cons:\nA) All of them: also add a 'Next: get a key and run examples/first_eval.py' line to the demo output (#5) and a shell-persistence note next to the export instruction (#7) (recommended) (human: ~1h / CC: ~5 min)\n ✅ The demo hands the developer directly to the live step instead of back to the README\n ✅ Closes the 'exported in the wrong shell' trap before it becomes an auth error\n ❌ Two more small edits to README and demo.py to coordinate with D4 and D9\nB) Let me pick which ones matter\n ✅ You choose exactly which of #5 and #7 join the plan\n ✅ Nothing lands you did not explicitly approve\n ❌ One more round trip for two small items\nC) Only the ones already decided (#1-#4, #6); skip #5 and #7\n ✅ No further edits beyond D4-D9\n ✅ The big blockers are all covered\n ❌ Demo output still ends without a next step; wrong-shell export still relies on the error message\nD) Unrealistic, our developers already know the flow\n ✅ No extra work\n ✅ Valid if the beta only goes to people who used 1.x\n ❌ The internal study measured a real developer at 6 minutes on this exact path\nNet: close the two small gaps now while README and demo.py are already open, or leave them for the post-beta survey to find.": "A) All, incl. #5 and #7 (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:18:24.904Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "df61df94-90b4-4edc-8a4d-4a354b9d774e",
|
||||
"toolUseId": "toolu_01QbMbsr3Mhx1EuW7tFiDTUc",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D11 — Pass 3/6: D5 made the binding check non-blocking but left its lifecycle unspecified. What happens when it fails, when the process exits first, and on fresh CI runners?\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: With D5, the first live evaluation returns its result right away while the sample-project binding check runs in the background. Three things are now undefined: (1) what the developer sees if that check later fails or times out; (2) what happens when a short script finishes before the check does, which is the normal case for a 5-second `first_eval.py`; (3) noninteractive CI mode, where every fresh runner counts as a 'first run', so the check fires on every pipeline. docs/current-contracts.md already has good text for the timeout (EVALKIT_CI_TIMEOUT, check URL, help link); the question is when and how often it appears, and whether anything is ever blocked again.\nStakes if we pick wrong: Too strict and the 5-minute wait sneaks back in through the CI door. Too loose and a real binding problem is printed once and never seen again. This is 'fight uncertainty' (developer must know whether it worked) balanced against 'zero friction'.\nRecommendation: B because a remembered warning keeps the problem visible on every later run without ever blocking a result, and it treats CI runners the same as laptops.\nCompleteness: A=7/10, B=10/10, C=5/10, D=6/10\nPros / cons:\nA) Advisory, fire-and-forget: print the warning if the check fails while the process is alive; if the process exits first, print one line with the check URL and exit normally; nothing is remembered (human: ~1 day / CC: ~15 min)\n ✅ Simplest possible semantics; never blocks, never re-nags\n ✅ Identical behavior on laptops and CI runners\n ❌ A failed binding is easy to miss: one line in a long log, and the next run says nothing\nB) Advisory and sticky: same as A, plus the SDK records a failed or unfinished check locally (config dir, or env var in CI) and each later live run prints one warning line with the check URL until a check passes; never blocks (recommended) (human: ~2 days / CC: ~25 min)\n ✅ The developer cannot lose track of a real binding problem, and still never waits for a result\n ✅ On CI, the warning shows up in every pipeline log until fixed, which is exactly where the team looks\n ❌ Needs a small local state file and its own tests (stale state, concurrent runs, read-only filesystems)\nC) Gate the second run: after a failed check, later live runs raise EVALKIT_CI_FAILED until it passes (human: ~1 day / CC: ~15 min)\n ✅ A broken binding cannot be ignored\n ✅ Reuses the existing timeout message content as an error\n ❌ Re-introduces a blocking gate one run later; a CI outage now breaks every developer's second run and every pipeline\nD) Wait at exit in noninteractive CI mode only: interactive runs behave as B, but in CI mode the process blocks at exit (bounded 300 s) so pipelines get a definitive pass/fail (human: ~2 days / CC: ~25 min)\n ✅ Pipelines get a hard answer instead of a warning\n ✅ Interactive TTHW unaffected\n ❌ Every fresh CI runner is a first run, so every pipeline pays up to 5 minutes; that is the original problem moved to CI\nNet: keep the check visible until it passes without ever putting it in front of a result, versus either forgetting failures or letting the wait back in through CI.",
|
||||
"header": "Check lifecycle",
|
||||
"options": [
|
||||
{
|
||||
"label": "B) Advisory and sticky (recommended)",
|
||||
"description": "Never blocks; failed/unfinished check is remembered and re-warned once per later live run until it passes."
|
||||
},
|
||||
{
|
||||
"label": "A) Advisory, fire-and-forget",
|
||||
"description": "Warn once if alive; print check URL at exit; nothing remembered."
|
||||
},
|
||||
{
|
||||
"label": "C) Gate the second run",
|
||||
"description": "Later live runs raise EVALKIT_CI_FAILED until the check passes."
|
||||
},
|
||||
{
|
||||
"label": "D) Wait at exit in CI mode",
|
||||
"description": "Interactive as B; noninteractive CI mode blocks at exit up to 300 s."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D11 — Pass 3/6: D5 made the binding check non-blocking but left its lifecycle unspecified. What happens when it fails, when the process exits first, and on fresh CI runners?\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan.\nELI10: With D5, the first live evaluation returns its result right away while the sample-project binding check runs in the background. Three things are now undefined: (1) what the developer sees if that check later fails or times out; (2) what happens when a short script finishes before the check does, which is the normal case for a 5-second `first_eval.py`; (3) noninteractive CI mode, where every fresh runner counts as a 'first run', so the check fires on every pipeline. docs/current-contracts.md already has good text for the timeout (EVALKIT_CI_TIMEOUT, check URL, help link); the question is when and how often it appears, and whether anything is ever blocked again.\nStakes if we pick wrong: Too strict and the 5-minute wait sneaks back in through the CI door. Too loose and a real binding problem is printed once and never seen again. This is 'fight uncertainty' (developer must know whether it worked) balanced against 'zero friction'.\nRecommendation: B because a remembered warning keeps the problem visible on every later run without ever blocking a result, and it treats CI runners the same as laptops.\nCompleteness: A=7/10, B=10/10, C=5/10, D=6/10\nPros / cons:\nA) Advisory, fire-and-forget: print the warning if the check fails while the process is alive; if the process exits first, print one line with the check URL and exit normally; nothing is remembered (human: ~1 day / CC: ~15 min)\n ✅ Simplest possible semantics; never blocks, never re-nags\n ✅ Identical behavior on laptops and CI runners\n ❌ A failed binding is easy to miss: one line in a long log, and the next run says nothing\nB) Advisory and sticky: same as A, plus the SDK records a failed or unfinished check locally (config dir, or env var in CI) and each later live run prints one warning line with the check URL until a check passes; never blocks (recommended) (human: ~2 days / CC: ~25 min)\n ✅ The developer cannot lose track of a real binding problem, and still never waits for a result\n ✅ On CI, the warning shows up in every pipeline log until fixed, which is exactly where the team looks\n ❌ Needs a small local state file and its own tests (stale state, concurrent runs, read-only filesystems)\nC) Gate the second run: after a failed check, later live runs raise EVALKIT_CI_FAILED until it passes (human: ~1 day / CC: ~15 min)\n ✅ A broken binding cannot be ignored\n ✅ Reuses the existing timeout message content as an error\n ❌ Re-introduces a blocking gate one run later; a CI outage now breaks every developer's second run and every pipeline\nD) Wait at exit in noninteractive CI mode only: interactive runs behave as B, but in CI mode the process blocks at exit (bounded 300 s) so pipelines get a definitive pass/fail (human: ~2 days / CC: ~25 min)\n ✅ Pipelines get a hard answer instead of a warning\n ✅ Interactive TTHW unaffected\n ❌ Every fresh CI runner is a first run, so every pipeline pays up to 5 minutes; that is the original problem moved to CI\nNet: keep the check visible until it passes without ever putting it in front of a result, versus either forgetting failures or letting the wait back in through CI.": "B) Advisory and sticky (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:20:13.670Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "df61df94-90b4-4edc-8a4d-4a354b9d774e",
|
||||
"toolUseId": "toolu_016HXc1Ct8mapiqhZ9T1REcj",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D12 — TODO candidate: publish a written deprecation policy alongside the 2.0 migration guide?\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan; TODO pass after the 8 DX passes.\nELI10: D8 gives `Client.evaluate()` an alias with a warning that says 'removal in 3.0'. That implies a rule (deprecated names live for one major version) but nothing writes the rule down. A developer deciding whether to trust EvalKit in production wants to read the policy once, not infer it from one warning.\nWhat: a short 'Deprecation policy' section in the changelog and docs: semantic versioning, deprecated APIs kept with a DeprecationWarning for at least one major version, removals announced in the changelog one release ahead, codemod shipped for every rename.\nWhy: Pass 5 scored 9/10; the missing point is that upgrade safety is demonstrated but not promised.\nPros: turns one good migration into a credible pattern; cheap; removes a recurring question from support.\nCons: a policy is a commitment the team must keep for every future rename.\nContext: lives next to the 'Migrating from 1.x' section from D8; can reuse the D8 warning text as its example.\nDepends on: D8 landing.\nStakes if we pick wrong: Small. Without it, each future deprecation is judged case by case by users.\nRecommendation: A because it is a paragraph of docs that makes D8's behavior a promise.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Add to TODOS.md (recommended)\n ✅ Captured with context so it can be written when the migration guide is drafted\n ✅ Does not add to the beta's critical path\n ❌ Could slip past the beta if nobody picks it up\nB) Skip\n ✅ No commitment made before the team agrees on a policy\n ✅ D8's warning already communicates the removal version\n ❌ Upgrade trust stays implicit\nC) Build it now (put it in this plan's implementation tasks) (human: ~1h / CC: ~5 min)\n ✅ Ships with the 2.0 changelog in the same edit as the migration guide\n ✅ One less loose end after beta\n ❌ Small scope addition to a POLISH-mode plan\nNet: write down the promise D8 already makes, now or later, or leave it implicit.",
|
||||
"header": "TODO",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Add to TODOS.md (recommended)",
|
||||
"description": "Capture as a follow-up with context."
|
||||
},
|
||||
{
|
||||
"label": "B) Skip",
|
||||
"description": "Do not track it."
|
||||
},
|
||||
{
|
||||
"label": "C) Build it now",
|
||||
"description": "Add as a P2 task in this plan alongside the D8 migration guide."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D12 — TODO candidate: publish a written deprecation policy alongside the 2.0 migration guide?\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan; TODO pass after the 8 DX passes.\nELI10: D8 gives `Client.evaluate()` an alias with a warning that says 'removal in 3.0'. That implies a rule (deprecated names live for one major version) but nothing writes the rule down. A developer deciding whether to trust EvalKit in production wants to read the policy once, not infer it from one warning.\nWhat: a short 'Deprecation policy' section in the changelog and docs: semantic versioning, deprecated APIs kept with a DeprecationWarning for at least one major version, removals announced in the changelog one release ahead, codemod shipped for every rename.\nWhy: Pass 5 scored 9/10; the missing point is that upgrade safety is demonstrated but not promised.\nPros: turns one good migration into a credible pattern; cheap; removes a recurring question from support.\nCons: a policy is a commitment the team must keep for every future rename.\nContext: lives next to the 'Migrating from 1.x' section from D8; can reuse the D8 warning text as its example.\nDepends on: D8 landing.\nStakes if we pick wrong: Small. Without it, each future deprecation is judged case by case by users.\nRecommendation: A because it is a paragraph of docs that makes D8's behavior a promise.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Add to TODOS.md (recommended)\n ✅ Captured with context so it can be written when the migration guide is drafted\n ✅ Does not add to the beta's critical path\n ❌ Could slip past the beta if nobody picks it up\nB) Skip\n ✅ No commitment made before the team agrees on a policy\n ✅ D8's warning already communicates the removal version\n ❌ Upgrade trust stays implicit\nC) Build it now (put it in this plan's implementation tasks) (human: ~1h / CC: ~5 min)\n ✅ Ships with the 2.0 changelog in the same edit as the migration guide\n ✅ One less loose end after beta\n ❌ Small scope addition to a POLISH-mode plan\nNet: write down the promise D8 already makes, now or later, or leave it implicit.": "A) Add to TODOS.md (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:20:47.421Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "df61df94-90b4-4edc-8a4d-4a354b9d774e",
|
||||
"toolUseId": "toolu_019XuebXaZYasJRDAqypAnqa",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D13 — TODO candidate: instrument the demo-to-first-live-run step so the drop-off between them is measurable?\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan; TODO pass after the 8 DX passes.\nELI10: docs/benchmarks.md says timing instrumentation already measures from install to first real evaluation result. After D4 and D10 the flow has two distinct successes: the keyless demo, then the keyed `first_eval.py`. Today's instrumentation cannot tell whether developers who finish the demo go on to create a key. That gap between the two is exactly where a 'free demo, then account' funnel loses people.\nWhat: emit two opt-in telemetry events, `demo_completed` and `first_live_eval_completed`, with the elapsed time between them, using the existing opt-in telemetry channel.\nWhy: Pass 8 scored 8/10; the missing points are journey analytics between the two milestones.\nPros: shows whether the key step is the new bottleneck once the CI wait is gone; gives the post-beta survey a number to correlate with.\nCons: telemetry is opt-in, so coverage will be partial; two more events to document in the privacy notes.\nContext: existing timing instrumentation and post-beta survey continue unchanged (docs/benchmarks.md:9-10); this adds two events to the same pipe.\nDepends on: D4 (first_eval.py exists) and D10 (demo prints the next step).\nStakes if we pick wrong: Small. Without it, the next benchmark run says '1.5 minutes' but cannot say how many people got to the second half.\nRecommendation: A because the target is under 2 minutes to a real result and this is how you learn if the key step now owns the remaining time.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Add to TODOS.md (recommended)\n ✅ Captured with the exact events and the reason, ready for whoever owns telemetry\n ✅ Keeps the beta plan focused on the contract repairs\n ❌ First beta cohort's demo-to-key conversion goes unmeasured\nB) Skip\n ✅ No telemetry surface change before beta\n ✅ The post-beta survey can ask the question qualitatively\n ❌ Survey answers are self-reported and lag by weeks\nC) Build it now (add as a P3 task in this plan) (human: ~half day / CC: ~10 min)\n ✅ The beta cohort is the best time to measure a brand-new funnel\n ✅ Two events on an existing opt-in pipe is a small change\n ❌ Adds a telemetry task to a POLISH-mode release plan\nNet: measure the new second half of onboarding, now or later, or rely on the survey.",
|
||||
"header": "TODO",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Add to TODOS.md (recommended)",
|
||||
"description": "Capture as a follow-up with the two event names."
|
||||
},
|
||||
{
|
||||
"label": "B) Skip",
|
||||
"description": "Do not track it."
|
||||
},
|
||||
{
|
||||
"label": "C) Build it now",
|
||||
"description": "Add as a P3 task in this plan."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D13 — TODO candidate: instrument the demo-to-first-live-run step so the drop-off between them is measurable?\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan; TODO pass after the 8 DX passes.\nELI10: docs/benchmarks.md says timing instrumentation already measures from install to first real evaluation result. After D4 and D10 the flow has two distinct successes: the keyless demo, then the keyed `first_eval.py`. Today's instrumentation cannot tell whether developers who finish the demo go on to create a key. That gap between the two is exactly where a 'free demo, then account' funnel loses people.\nWhat: emit two opt-in telemetry events, `demo_completed` and `first_live_eval_completed`, with the elapsed time between them, using the existing opt-in telemetry channel.\nWhy: Pass 8 scored 8/10; the missing points are journey analytics between the two milestones.\nPros: shows whether the key step is the new bottleneck once the CI wait is gone; gives the post-beta survey a number to correlate with.\nCons: telemetry is opt-in, so coverage will be partial; two more events to document in the privacy notes.\nContext: existing timing instrumentation and post-beta survey continue unchanged (docs/benchmarks.md:9-10); this adds two events to the same pipe.\nDepends on: D4 (first_eval.py exists) and D10 (demo prints the next step).\nStakes if we pick wrong: Small. Without it, the next benchmark run says '1.5 minutes' but cannot say how many people got to the second half.\nRecommendation: A because the target is under 2 minutes to a real result and this is how you learn if the key step now owns the remaining time.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Add to TODOS.md (recommended)\n ✅ Captured with the exact events and the reason, ready for whoever owns telemetry\n ✅ Keeps the beta plan focused on the contract repairs\n ❌ First beta cohort's demo-to-key conversion goes unmeasured\nB) Skip\n ✅ No telemetry surface change before beta\n ✅ The post-beta survey can ask the question qualitatively\n ❌ Survey answers are self-reported and lag by weeks\nC) Build it now (add as a P3 task in this plan) (human: ~half day / CC: ~10 min)\n ✅ The beta cohort is the best time to measure a brand-new funnel\n ✅ Two events on an existing opt-in pipe is a small change\n ❌ Adds a telemetry task to a POLISH-mode release plan\nNet: measure the new second half of onboarding, now or later, or rely on the survey.": "A) Add to TODOS.md (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:21:08.069Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "df61df94-90b4-4edc-8a4d-4a354b9d774e",
|
||||
"toolUseId": "toolu_011ovc1ZRdT3t3d9zfED2oaF",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D14 — TODO candidate: audit every SDK error for a stable error code and a docs link, matching the new auth and swapped-args errors?\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan; TODO pass after the 8 DX passes.\nELI10: docs/current-contracts.md says all non-auth errors already state the cause, the argument or file, and a fix. It does not say they carry a stable code (like EVALKIT_CI_TIMEOUT does) or a link to a docs page. After D6 and D7 there will be three errors with codes and links and an unknown number without. A developer who greps logs for EVALKIT_ codes, or clicks the link in one error and finds none in the next, notices the inconsistency.\nWhat: audit the error classes in evalkit/client.py, give each a stable `code` attribute and `doc_url`, and add one docs page listing every code with its cause and fix (the Rust error-index pattern).\nWhy: Pass 3 scored 9/10 after D7; the last point is uniformity across the whole surface, which cannot be verified from the planning repo alone.\nPros: every error becomes searchable and linkable; support and CI log triage get a shared vocabulary.\nCons: touches every error site; the docs page must be kept in sync with new codes.\nContext: the runtime lives in a separate repo (README.md:39-40), so the audit needs that checkout. D6 and D7 set the pattern to follow.\nDepends on: D6, D7 landing first so the pattern exists.\nStakes if we pick wrong: Small. Errors already say what went wrong; this adds consistency and findability.\nRecommendation: A because it is real DX debt that this repo cannot verify or fix, so it belongs in TODOS.md with the runtime repo named.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Add to TODOS.md (recommended)\n ✅ Recorded with the pattern to follow and the repo it lives in\n ✅ Does not block the beta on an audit of code this plan cannot see\n ❌ Codes stay inconsistent across errors for the beta period\nB) Skip\n ✅ Errors already meet the cause-plus-fix bar per the contracts doc\n ✅ No cross-repo work implied by this plan\n ❌ The new coded errors make the uncoded ones look like the odd ones out\nC) Build it now (add as a P3 task in this plan) (human: ~2 days / CC: ~30 min)\n ✅ Beta ships with one consistent error contract\n ✅ Docs error index becomes a support asset from day one\n ❌ Requires the runtime repo and widens a POLISH-mode plan\nNet: track the uniformity work where it can actually be done, or leave the three new errors as the only coded ones.",
|
||||
"header": "TODO",
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Add to TODOS.md (recommended)",
|
||||
"description": "Capture as a follow-up naming the runtime repo and the D6/D7 pattern."
|
||||
},
|
||||
{
|
||||
"label": "B) Skip",
|
||||
"description": "Do not track it."
|
||||
},
|
||||
{
|
||||
"label": "C) Build it now",
|
||||
"description": "Add as a P3 task in this plan."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D14 — TODO candidate: audit every SDK error for a stable error code and a docs link, matching the new auth and swapped-args errors?\nProject/branch/task: gstack-plan-count-ig4gtr on main, /plan-devex-review of the EvalKit 2.0.0b1 beta plan; TODO pass after the 8 DX passes.\nELI10: docs/current-contracts.md says all non-auth errors already state the cause, the argument or file, and a fix. It does not say they carry a stable code (like EVALKIT_CI_TIMEOUT does) or a link to a docs page. After D6 and D7 there will be three errors with codes and links and an unknown number without. A developer who greps logs for EVALKIT_ codes, or clicks the link in one error and finds none in the next, notices the inconsistency.\nWhat: audit the error classes in evalkit/client.py, give each a stable `code` attribute and `doc_url`, and add one docs page listing every code with its cause and fix (the Rust error-index pattern).\nWhy: Pass 3 scored 9/10 after D7; the last point is uniformity across the whole surface, which cannot be verified from the planning repo alone.\nPros: every error becomes searchable and linkable; support and CI log triage get a shared vocabulary.\nCons: touches every error site; the docs page must be kept in sync with new codes.\nContext: the runtime lives in a separate repo (README.md:39-40), so the audit needs that checkout. D6 and D7 set the pattern to follow.\nDepends on: D6, D7 landing first so the pattern exists.\nStakes if we pick wrong: Small. Errors already say what went wrong; this adds consistency and findability.\nRecommendation: A because it is real DX debt that this repo cannot verify or fix, so it belongs in TODOS.md with the runtime repo named.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Add to TODOS.md (recommended)\n ✅ Recorded with the pattern to follow and the repo it lives in\n ✅ Does not block the beta on an audit of code this plan cannot see\n ❌ Codes stay inconsistent across errors for the beta period\nB) Skip\n ✅ Errors already meet the cause-plus-fix bar per the contracts doc\n ✅ No cross-repo work implied by this plan\n ❌ The new coded errors make the uncoded ones look like the odd ones out\nC) Build it now (add as a P3 task in this plan) (human: ~2 days / CC: ~30 min)\n ✅ Beta ships with one consistent error contract\n ✅ Docs error index becomes a support asset from day one\n ❌ Requires the runtime repo and widens a POLISH-mode plan\nNet: track the uniformity work where it can actually be done, or leave the three new errors as the only coded ones.": "A) Add to TODOS.md (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:21:30.769Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
+204
@@ -0,0 +1,204 @@
|
||||
{
|
||||
"provenance": {
|
||||
"sourceHead": "f26d569e0345cb1131d9ca52d4a43965085c3468",
|
||||
"sourceObservationSha256": "19fe5cb03390e626a239cfea2a257477343d4c63cba13405348bb61e1b6b3107",
|
||||
"publicNativeProofSha256": "df6a8eabe49dcdd1743abedccf891cc631456c4989a63758bab4fdccf12dcca9",
|
||||
"historicalOutcome": "plan_ready; local-ci-gate, missing-quickstart and reversed-arguments seed predicates failed",
|
||||
"paidOutcomesReclassified": false,
|
||||
"scope": "Exact completed first-attempt native questions, options, answers and acknowledgment times only; pending/retry calls are not included."
|
||||
},
|
||||
"calls": [
|
||||
{
|
||||
"sessionId": "8c7b28d3-987f-4387-81fb-ea3703c6cac4",
|
||||
"toolUseId": "toolu_01TUskvznLiAKducCZebGFeH",
|
||||
"questions": [
|
||||
{
|
||||
"question": "gstack works best when your project's CLAUDE.md includes skill routing rules. Add them? (One-time prompt per project. Note: plan mode is active, so if you pick A the edit and commit happen after this review exits plan mode.)",
|
||||
"header": "Routing",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Add routing rules to CLAUDE.md (recommended)",
|
||||
"description": "Append a '## Skill routing' section mapping request types to gstack skills, then commit it. Applied after plan mode exits."
|
||||
},
|
||||
{
|
||||
"label": "No thanks, I'll invoke skills manually",
|
||||
"description": "Record routing_declined=true so this prompt does not recur for this project. Re-enable later via gstack-config."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"gstack works best when your project's CLAUDE.md includes skill routing rules. Add them? (One-time prompt per project. Note: plan mode is active, so if you pick A the edit and commit happen after this review exits plan mode.)": "Add routing rules to CLAUDE.md (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:02:43.124Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "8c7b28d3-987f-4387-81fb-ea3703c6cac4",
|
||||
"toolUseId": "toolu_01RMPrDBdVYuxNKXQCCD4sY3",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D1 — Mandatory 5-minute remote CI check before the first local result\nProject/branch/task: EvalKit SDK beta on main; DX POLISH review of the first-run contract.\nELI10: Today the very first evaluation, including the keyless `python -m evalkit.demo` that uses a mock transport and bundled data, cannot return until a remote CI check succeeds, and that check blocks for up to 300 seconds with no skip or offline path (docs/current-contracts.md lines 3-5). The onboarding study clocks EvalKit at 6 minutes, of which 5 is this wait; the agreed target is under 2 minutes and the fastest peer takes 2. The persona explicitly wants a local result BEFORE connecting to production CI, so the gate sits on the exact moment the demo is supposed to feel like magic.\nStakes if we pick wrong: the magical moment arrives five minutes late, a keyless demo fails outright on machines without network or CI access, and EvalKit stays last in its own benchmark.\nRecommendation: A because the persona's stated need is local-first, and a mock-transport demo has no reason to depend on remote CI; the existing progress and EVALKIT_CI_TIMEOUT messaging stays intact for the step where CI verification actually belongs.\nCompleteness: A=10/10, B=8/10, C=6/10, D=2/10\nNet: trading a first-run gate that verifies the sample-project binding against a first result that lands in seconds; the check still exists, it just stops standing between the developer and their first number.",
|
||||
"header": "First-run CI",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Remove gate from local runs and demo (recommended)",
|
||||
"description": "✅ Demo and first local eval return in seconds; TTHW drops from 6 min to roughly 1 min, beating every peer. ✅ Keyless demo works offline and in sandboxed CI where the remote check cannot reach. ✅ Existing CI-check messaging (progress line, check URL, EVALKIT_CI_TIMEOUT) is kept for an explicit opt-in step when the developer connects to production CI. ❌ The sample-project binding is no longer verified on the very first run; misconfigured bindings surface at the CI-connection step instead. (human: ~1 day / CC: ~20 min)"
|
||||
},
|
||||
{
|
||||
"label": "Keep check, make it non-blocking",
|
||||
"description": "✅ Local result prints immediately while the CI check runs in the background and reports when it finishes. ✅ Binding is still verified on first run, so misconfiguration is caught as early as today. ❌ Background network activity during a keyless offline demo is surprising, and a failed background check still needs a reporting path the developer may never see. (human: ~2 days / CC: ~30 min)"
|
||||
},
|
||||
{
|
||||
"label": "Keep gate, add an explicit skip flag",
|
||||
"description": "✅ Existing contract is untouched for developers who want the verification. ✅ An escape hatch such as --skip-ci-check or EVALKIT_OFFLINE=1 lets the demo run offline. ❌ The default path still waits 5 minutes, so the study number does not move unless the developer knows the flag exists, which violates zero friction at T0. (human: ~half day / CC: ~10 min)"
|
||||
},
|
||||
{
|
||||
"label": "Keep as-is",
|
||||
"description": "✅ Zero implementation cost for the beta. ✅ Every first run is verified against CI before any result is shown. ❌ TTHW stays at 6 minutes against an agreed target of under 2, and the demo fails without network access."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D1 — Mandatory 5-minute remote CI check before the first local result\nProject/branch/task: EvalKit SDK beta on main; DX POLISH review of the first-run contract.\nELI10: Today the very first evaluation, including the keyless `python -m evalkit.demo` that uses a mock transport and bundled data, cannot return until a remote CI check succeeds, and that check blocks for up to 300 seconds with no skip or offline path (docs/current-contracts.md lines 3-5). The onboarding study clocks EvalKit at 6 minutes, of which 5 is this wait; the agreed target is under 2 minutes and the fastest peer takes 2. The persona explicitly wants a local result BEFORE connecting to production CI, so the gate sits on the exact moment the demo is supposed to feel like magic.\nStakes if we pick wrong: the magical moment arrives five minutes late, a keyless demo fails outright on machines without network or CI access, and EvalKit stays last in its own benchmark.\nRecommendation: A because the persona's stated need is local-first, and a mock-transport demo has no reason to depend on remote CI; the existing progress and EVALKIT_CI_TIMEOUT messaging stays intact for the step where CI verification actually belongs.\nCompleteness: A=10/10, B=8/10, C=6/10, D=2/10\nNet: trading a first-run gate that verifies the sample-project binding against a first result that lands in seconds; the check still exists, it just stops standing between the developer and their first number.": "Remove gate from local runs and demo (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:05:39.125Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "8c7b28d3-987f-4387-81fb-ea3703c6cac4",
|
||||
"toolUseId": "toolu_01QhJNW7VX12xDdTqPKfqQKy",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D2 — run_eval(dataset, evaluator) vs run_batch(evaluator, dataset): reversed positional order\nProject/branch/task: EvalKit SDK beta on main; DX POLISH review of the public function signatures in docs/api.md.\nELI10: The two evaluation functions take the same two things, a dataset and an evaluator, but in opposite positional order, and neither requires keyword arguments (docs/api.md lines 3-9). An ML engineer who learns run_eval and then calls run_batch the same way passes the arguments swapped. Nothing in the signature stops them, so the mistake shows up at runtime, possibly as a confusing type error deep inside the evaluator. The Hall of Fame test for API design is 'can the persona use it correctly after seeing one example'; this pair fails that test by construction.\nStakes if we pick wrong: every developer who scales from single eval to batch trips on the swap, and once 2.0 final ships the order becomes a compatibility promise that is expensive to fix.\nRecommendation: A because 2.0.0b1 is a beta, so aligning the order now costs almost nothing, and a swapped-type check turns the remaining mistake into a one-line fix message instead of a stack trace.\nCompleteness: A=10/10, B=8/10, C=2/10\nNet: trading a small beta-only signature change against a permanent trap in the two most-called functions of the SDK.",
|
||||
"header": "Signatures",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Align order to (dataset, evaluator) + swap detection (recommended)",
|
||||
"description": "✅ Both functions read the same way, so one example teaches both; matches the mental model 'evaluate this dataset with this evaluator'. ✅ If a developer still passes them swapped, the SDK detects the types and raises a message naming the fix: 'run_batch received an evaluator where the dataset goes; call run_batch(dataset, evaluator)'. ✅ Changelog carries a one-line migration note for anyone on an earlier beta. ❌ Anyone who wrote against the b1 order of run_batch must flip two arguments once. (human: ~half day / CC: ~15 min)"
|
||||
},
|
||||
{
|
||||
"label": "Make both keyword-only",
|
||||
"description": "✅ run_eval(dataset=..., evaluator=...) makes a swap impossible at every call site. ✅ Self-documenting at the call site without opening the docs. ❌ More typing for the common two-argument call, and it is a larger break for anyone already using positional calls in either function. (human: ~half day / CC: ~15 min)"
|
||||
},
|
||||
{
|
||||
"label": "Keep reversed order as documented",
|
||||
"description": "✅ No signature change and no migration note for the beta. ✅ Existing internal callers keep working untouched. ❌ The trap ships into 2.0 final where it becomes a compatibility promise, and the docs must carry a permanent warning about the reversed order."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D2 — run_eval(dataset, evaluator) vs run_batch(evaluator, dataset): reversed positional order\nProject/branch/task: EvalKit SDK beta on main; DX POLISH review of the public function signatures in docs/api.md.\nELI10: The two evaluation functions take the same two things, a dataset and an evaluator, but in opposite positional order, and neither requires keyword arguments (docs/api.md lines 3-9). An ML engineer who learns run_eval and then calls run_batch the same way passes the arguments swapped. Nothing in the signature stops them, so the mistake shows up at runtime, possibly as a confusing type error deep inside the evaluator. The Hall of Fame test for API design is 'can the persona use it correctly after seeing one example'; this pair fails that test by construction.\nStakes if we pick wrong: every developer who scales from single eval to batch trips on the swap, and once 2.0 final ships the order becomes a compatibility promise that is expensive to fix.\nRecommendation: A because 2.0.0b1 is a beta, so aligning the order now costs almost nothing, and a swapped-type check turns the remaining mistake into a one-line fix message instead of a stack trace.\nCompleteness: A=10/10, B=8/10, C=2/10\nNet: trading a small beta-only signature change against a permanent trap in the two most-called functions of the SDK.": "Align order to (dataset, evaluator) + swap detection (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:06:04.706Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "8c7b28d3-987f-4387-81fb-ea3703c6cac4",
|
||||
"toolUseId": "toolu_01LfVsMSQhhnfMjM6NGT9oan",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D3 — Invalid API key raises AuthError(\"request failed\") with no code, cause, or fix\nProject/branch/task: EvalKit SDK beta on main; DX POLISH review of the authentication error in docs/api.md.\nELI10: Right after the demo, the persona creates a key in the console, copies it once, and exports EVALKIT_API_KEY. If they paste it wrong, export it in a different shell, or the key was revoked, the SDK raises AuthError(\"request failed\") (docs/api.md lines 11-13). That message does not say it was an auth problem, does not say which key was tried, and does not say where to get a new one. Every other EvalKit error already names the cause, the relevant argument, and a fix (docs/current-contracts.md lines 21-24); this one error is the outlier, and it sits at the first moment the developer moves from demo to real usage.\nStakes if we pick wrong: the developer's first live evaluation fails with a message that looks like a network fault, they debug the wrong thing, and the second minute after the magical moment becomes the drop-off point.\nRecommendation: A because the SDK already has the structured-error pattern for every other failure; the auth error just needs to join it, and it is the highest-traffic error a new developer will ever see.\nCompleteness: A=10/10, B=6/10, C=1/10\nNet: trading a small error-class change against the single most common first-live-call failure being undiagnosable.",
|
||||
"header": "Auth error",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Structured auth error: code + cause + fix + link (recommended)",
|
||||
"description": "✅ Message follows the Stripe formula: 'EVALKIT_AUTH_INVALID_KEY: the API key from EVALKIT_API_KEY (ending ...ab12) was rejected by EvalKit. Create or rotate a key at https://console.evalkit.example/settings/api-keys and re-export EVALKIT_API_KEY.' ✅ Distinguishes the three real cases: key missing (env var unset), key invalid, key revoked or wrong project; each gets its own code and fix line. ✅ Error object exposes .code and .doc_url like the other typed errors, and the key is redacted to its last 4 characters per the existing redaction contract. ❌ Three codes means three messages to write and test instead of one. (human: ~half day / CC: ~15 min)"
|
||||
},
|
||||
{
|
||||
"label": "Improve the message text only",
|
||||
"description": "✅ Cheapest change: replace 'request failed' with 'Invalid API key; check EVALKIT_API_KEY' and ship. ✅ No new error codes or attributes to document. ❌ Still no code to search for, no distinction between missing and invalid, and no link to where a key is created, so the developer leaves the terminal to hunt for the console page. (human: ~1 hour / CC: ~5 min)"
|
||||
},
|
||||
{
|
||||
"label": "Keep AuthError(\"request failed\")",
|
||||
"description": "✅ No change to the beta error surface. ✅ Zero implementation cost. ❌ The one error every new developer is likely to hit is the only one in the SDK that violates the documented problem + cause + fix contract."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D3 — Invalid API key raises AuthError(\"request failed\") with no code, cause, or fix\nProject/branch/task: EvalKit SDK beta on main; DX POLISH review of the authentication error in docs/api.md.\nELI10: Right after the demo, the persona creates a key in the console, copies it once, and exports EVALKIT_API_KEY. If they paste it wrong, export it in a different shell, or the key was revoked, the SDK raises AuthError(\"request failed\") (docs/api.md lines 11-13). That message does not say it was an auth problem, does not say which key was tried, and does not say where to get a new one. Every other EvalKit error already names the cause, the relevant argument, and a fix (docs/current-contracts.md lines 21-24); this one error is the outlier, and it sits at the first moment the developer moves from demo to real usage.\nStakes if we pick wrong: the developer's first live evaluation fails with a message that looks like a network fault, they debug the wrong thing, and the second minute after the magical moment becomes the drop-off point.\nRecommendation: A because the SDK already has the structured-error pattern for every other failure; the auth error just needs to join it, and it is the highest-traffic error a new developer will ever see.\nCompleteness: A=10/10, B=6/10, C=1/10\nNet: trading a small error-class change against the single most common first-live-call failure being undiagnosable.": "Structured auth error: code + cause + fix + link (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:06:30.294Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "8c7b28d3-987f-4387-81fb-ea3703c6cac4",
|
||||
"toolUseId": "toolu_01Kc3x4EvYWAkRiBYMwGfySU",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D4 — README quickstart points at examples/first_eval.py, which is not in the package or the examples archive\nProject/branch/task: EvalKit SDK beta on main; DX POLISH review of the packaged quickstart in README.md and docs/package-contents.txt.\nELI10: README.md line 11 tells the developer to run `python examples/first_eval.py` right after install. docs/package-contents.txt lines 8-9 confirm that file is absent from both the published wheel and the release examples archive. The only shipped runnable entry point is `python -m evalkit.demo` (evalkit/demo.py plus sample_responses.json), which is also the agreed magical-moment vehicle. So the first command the persona is told to type fails with 'No such file or directory' in minute one, and the command that actually works is mentioned as an afterthought three lines later.\nStakes if we pick wrong: the very first instruction in the quickstart is broken, which is the classic minute-two abandonment; even developers who recover now have two competing 'first commands', which the Hall of Fame flags as decision fatigue.\nRecommendation: A because the demo already exists, already works keyless, and is the chosen vehicle; making it the one quickstart command removes the broken reference and the fork in the road at once, and the live-eval example then has a clear home as step two.\nCompleteness: A=10/10, B=8/10, C=5/10\nNet: trading a README rewrite plus one shipped example module against a quickstart whose first line fails.",
|
||||
"header": "Quickstart",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Quickstart = the demo; ship the live example as step two (recommended)",
|
||||
"description": "✅ Step 1 becomes `python -m evalkit.demo` with the documented expected output right below it, so install-to-scores is two commands and one golden path. ✅ Step 2 (after the key) runs a shipped module, `python -m evalkit.examples.first_eval`, so the file travels inside the wheel and can never be missing again; package-contents.txt is updated to list it. ✅ The stale examples/first_eval.py reference is removed from README and the examples archive gets the same file for people who browse the repo. ❌ One new module to write, test, and keep in sync with the demo output. (human: ~half day / CC: ~15 min)"
|
||||
},
|
||||
{
|
||||
"label": "Add examples/first_eval.py to package and archive",
|
||||
"description": "✅ The README command starts working exactly as written, with no doc rewrite. ✅ Developers who cloned the repo find the file where the README says. ❌ A top-level examples/ path only works from a source checkout, not after pip install, so the quickstart still fails for the pip-install persona; and the demo remains a second, competing first command. (human: ~2 hours / CC: ~10 min)"
|
||||
},
|
||||
{
|
||||
"label": "Remove the reference only",
|
||||
"description": "✅ Fastest fix: delete line 11 so the quickstart no longer points at a missing file. ✅ Zero packaging changes. ❌ There is then no packaged example of a live evaluation with a real key, so the jump from demo to real usage is undocumented. (human: ~15 min / CC: ~2 min)"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D4 — README quickstart points at examples/first_eval.py, which is not in the package or the examples archive\nProject/branch/task: EvalKit SDK beta on main; DX POLISH review of the packaged quickstart in README.md and docs/package-contents.txt.\nELI10: README.md line 11 tells the developer to run `python examples/first_eval.py` right after install. docs/package-contents.txt lines 8-9 confirm that file is absent from both the published wheel and the release examples archive. The only shipped runnable entry point is `python -m evalkit.demo` (evalkit/demo.py plus sample_responses.json), which is also the agreed magical-moment vehicle. So the first command the persona is told to type fails with 'No such file or directory' in minute one, and the command that actually works is mentioned as an afterthought three lines later.\nStakes if we pick wrong: the very first instruction in the quickstart is broken, which is the classic minute-two abandonment; even developers who recover now have two competing 'first commands', which the Hall of Fame flags as decision fatigue.\nRecommendation: A because the demo already exists, already works keyless, and is the chosen vehicle; making it the one quickstart command removes the broken reference and the fork in the road at once, and the live-eval example then has a clear home as step two.\nCompleteness: A=10/10, B=8/10, C=5/10\nNet: trading a README rewrite plus one shipped example module against a quickstart whose first line fails.": "Quickstart = the demo; ship the live example as step two (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:06:59.908Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "8c7b28d3-987f-4387-81fb-ea3703c6cac4",
|
||||
"toolUseId": "toolu_01DYFQofwAoRqDeZ6HTxzjyP",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D5 — v2 removes Client.evaluate() immediately with no alias, deprecation warning, migration guide, or codemod\nProject/branch/task: EvalKit SDK beta on main; DX POLISH review of the v1-to-v2 client upgrade in docs/api.md.\nELI10: Version 1 users call Client.evaluate(). Version 2 renames it to Client.run() and deletes the old name on the spot (docs/api.md lines 15-18). A v1 team that bumps to 2.0 sees AttributeError: 'Client' object has no attribute 'evaluate' at runtime, possibly in production CI, with nothing pointing them to run(). The changelog is otherwise complete, so this rename is the only breaking change without a path. Upgrades should be boring; this one is a surprise.\nStakes if we pick wrong: existing v1 users, the people most likely to champion the SDK, get a red CI run on upgrade and learn that EvalKit majors break without warning, which poisons every future upgrade.\nRecommendation: A because a deprecated alias costs a few lines, gives v1 users one release of warning with the exact replacement in the message, and lets the rename land cleanly in 3.0.\nCompleteness: A=10/10, B=6/10, C=1/10\nNet: trading a few lines of alias plus a changelog section against v1 users discovering the rename via a production AttributeError.",
|
||||
"header": "v1 to v2",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Deprecated alias + warning + migration guide; remove in 3.0 (recommended)",
|
||||
"description": "✅ Client.evaluate() keeps working in 2.x and emits DeprecationWarning: 'Client.evaluate() is deprecated and will be removed in 3.0; use Client.run(), same arguments.' ✅ Changelog gains a 'Migrating from 1.x' section with the one-line change, plus a copy-paste sed or one-file codemod that rewrites .evaluate( to .run( in a repo. ✅ v1 users upgrade with green CI and fix at their pace; removal is announced one major ahead. ❌ One deprecated symbol lives in the codebase for the 2.x line. (human: ~half day / CC: ~15 min)"
|
||||
},
|
||||
{
|
||||
"label": "Remove, but ship a migration guide and a pointing error",
|
||||
"description": "✅ Clean 2.0 surface with no legacy symbol. ✅ Accessing Client.evaluate raises a typed error naming run() and linking the migration guide instead of a bare AttributeError. ❌ Upgrading still breaks v1 code at runtime; the developer only learns the fix after the failure rather than before. (human: ~2 hours / CC: ~10 min)"
|
||||
},
|
||||
{
|
||||
"label": "Keep the immediate removal as drafted",
|
||||
"description": "✅ Zero implementation cost and the smallest 2.0 API surface. ✅ No deprecation lifecycle to manage. ❌ v1 users get an unexplained AttributeError on upgrade with no guide, the one breaking change the otherwise complete changelog does not cover."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D5 — v2 removes Client.evaluate() immediately with no alias, deprecation warning, migration guide, or codemod\nProject/branch/task: EvalKit SDK beta on main; DX POLISH review of the v1-to-v2 client upgrade in docs/api.md.\nELI10: Version 1 users call Client.evaluate(). Version 2 renames it to Client.run() and deletes the old name on the spot (docs/api.md lines 15-18). A v1 team that bumps to 2.0 sees AttributeError: 'Client' object has no attribute 'evaluate' at runtime, possibly in production CI, with nothing pointing them to run(). The changelog is otherwise complete, so this rename is the only breaking change without a path. Upgrades should be boring; this one is a surprise.\nStakes if we pick wrong: existing v1 users, the people most likely to champion the SDK, get a red CI run on upgrade and learn that EvalKit majors break without warning, which poisons every future upgrade.\nRecommendation: A because a deprecated alias costs a few lines, gives v1 users one release of warning with the exact replacement in the message, and lets the rename land cleanly in 3.0.\nCompleteness: A=10/10, B=6/10, C=1/10\nNet: trading a few lines of alias plus a changelog section against v1 users discovering the rename via a production AttributeError.": "Deprecated alias + warning + migration guide; remove in 3.0 (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:07:27.548Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"provenance": {
|
||||
"run": "ship-source-as-delta-paid-20260910-v1",
|
||||
"attempt": "retry",
|
||||
"outcome": "no_review_questions",
|
||||
"retrospectivePass": false,
|
||||
"publicSource": "completed native AskUserQuestion with exact acknowledgment",
|
||||
"sourceProof": ".context/source-as-eng-retry-terminal-public-native-v1/proof.json"
|
||||
},
|
||||
"call": {
|
||||
"sessionId": "9e247dab-a089-4c51-ad6d-f067e2d4e5fe",
|
||||
"toolUseId": "toolu_014xtoxbQBBjjSJV7QxaiS1N",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D2 \u2014 Who is allowed to write to the auth cache?\nProject/branch/task: main \u2014 Multi-tenant Auth Refactor, AuthBroker and SessionMint both mutating one backing cache (PLAN.md:10, 19-20).\nELI10: Two services write to the same cache and nothing orders their writes. If one service removes a revoked token while the other is mid-way through storing that same token, the store lands last and the revoked token comes back to life. Users would see a logged-out or revoked session keep working.\nStakes if we pick wrong: A revoked or suspended tenant's tokens stay valid until natural expiry, silently, with no log line to find it.\nRecommendation: 2A because one writer plus a version check makes the race structurally impossible instead of merely unlikely; explicit over clever.\nCompleteness: 2A=10/10, 2B=7/10, 2C=3/10\nPros / cons:\n2A) Single writer + versioned writes: only SessionMint writes; AuthBroker reads and calls the adapter's existing invalidation hooks; every write carries the policy version read at validation start and the adapter rejects writes whose version is stale (human: ~1.5 days / CC: ~25 min) (recommended)\n \u2705 Revocation racing a mint can never resurrect a token; the stale write is rejected and logged\n \u2705 Ownership is obvious from the code: one class writes, one class reads, testable with a fake adapter\n \u274c Needs a compare-and-set or version-tag on the adapter write path, a small adapter change\n2B) Single writer only: SessionMint writes, AuthBroker reads and invalidates, no version check (human: ~1 day / CC: ~15 min)\n \u2705 Removes the two-writer coupling with no adapter change at all\n \u2705 Simpler to explain and diagram than a version scheme\n \u274c Invalidate-then-write-back is still possible inside a single mint that started before the revocation\n2C) Keep both writers, document the race as accepted (human: ~0 / CC: ~0)\n \u2705 Zero implementation cost right now\n \u2705 Matches the plan as written\n \u274c Ships a known fail-open path in auth; the first incident is a security incident\nNet: You are trading a small adapter change for a guarantee that revocation always wins the race.",
|
||||
"header": "Cache writes",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "2A Single writer + version (recommended)",
|
||||
"description": "SessionMint writes with policy-version tag; adapter rejects stale writes; AuthBroker reads/invalidates."
|
||||
},
|
||||
{
|
||||
"label": "2B Single writer only",
|
||||
"description": "SessionMint writes, AuthBroker reads and invalidates. No version check."
|
||||
},
|
||||
{
|
||||
"label": "2C Accept the race",
|
||||
"description": "Keep both writers as planned and document the known race."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D2 \u2014 Who is allowed to write to the auth cache?\nProject/branch/task: main \u2014 Multi-tenant Auth Refactor, AuthBroker and SessionMint both mutating one backing cache (PLAN.md:10, 19-20).\nELI10: Two services write to the same cache and nothing orders their writes. If one service removes a revoked token while the other is mid-way through storing that same token, the store lands last and the revoked token comes back to life. Users would see a logged-out or revoked session keep working.\nStakes if we pick wrong: A revoked or suspended tenant's tokens stay valid until natural expiry, silently, with no log line to find it.\nRecommendation: 2A because one writer plus a version check makes the race structurally impossible instead of merely unlikely; explicit over clever.\nCompleteness: 2A=10/10, 2B=7/10, 2C=3/10\nPros / cons:\n2A) Single writer + versioned writes: only SessionMint writes; AuthBroker reads and calls the adapter's existing invalidation hooks; every write carries the policy version read at validation start and the adapter rejects writes whose version is stale (human: ~1.5 days / CC: ~25 min) (recommended)\n \u2705 Revocation racing a mint can never resurrect a token; the stale write is rejected and logged\n \u2705 Ownership is obvious from the code: one class writes, one class reads, testable with a fake adapter\n \u274c Needs a compare-and-set or version-tag on the adapter write path, a small adapter change\n2B) Single writer only: SessionMint writes, AuthBroker reads and invalidates, no version check (human: ~1 day / CC: ~15 min)\n \u2705 Removes the two-writer coupling with no adapter change at all\n \u2705 Simpler to explain and diagram than a version scheme\n \u274c Invalidate-then-write-back is still possible inside a single mint that started before the revocation\n2C) Keep both writers, document the race as accepted (human: ~0 / CC: ~0)\n \u2705 Zero implementation cost right now\n \u2705 Matches the plan as written\n \u274c Ships a known fail-open path in auth; the first incident is a security incident\nNet: You are trading a small adapter change for a guarantee that revocation always wins the race.": "2A Single writer + version (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:25:21.601Z"
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"provenance": {
|
||||
"run": "ship-source-as-delta-paid-20260910-v1",
|
||||
"attempt": "first",
|
||||
"outcome": "no_review_questions",
|
||||
"retrospectivePass": false,
|
||||
"publicSource": "completed native AskUserQuestion and acknowledged Write"
|
||||
},
|
||||
"call": {
|
||||
"sessionId": "702366df-907f-4db0-aa9f-8b1444f340c5",
|
||||
"toolUseId": "toolu_01N6wYBzrBc23yXYTTTsy29o",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D4 \u2014 Issue 1 [P1] (confidence 9/10) PLAN.md:19-20: AuthBroker and SessionMint share a global mutable AuthCache via module-level export, and both mutate it.\nProject/branch/task: gstack-plan-count on main \u2014 Section 1 Architecture, Multi-tenant Auth Refactor.\nELI10: Two services write to the same cache through a global variable. The plan also says the cache 'does not serialize mutations' (PLAN.md:10). So writes can interleave: SessionMint stores a session while AuthBroker invalidates the same tenant on logout, and the last writer wins with no ordering. A module-level export also means tests share one instance, so one test's leftover token can make another pass or fail. Search check [Layer 1]: dependency injection is the standard cure; module singletons are the documented trap.\nStakes if we pick wrong: a revoked token stays valid for one tenant because a concurrent mint re-wrote it, and no test can reproduce it because the shared instance hides ordering.\nRecommendation: 1A because a single instance is fine, a global accessor is not. Inject it once from the composition root and give the facade a narrow write API that owns ordering. Maps to your 'explicit over clever' preference.\nCompleteness: A=10/10, B=6/10, C=2/10.\nNet: one extra constructor parameter per service buys deterministic writes and isolated tests.",
|
||||
"header": "Shared cache",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "1A) Inject AuthCache; narrow write API with per-key ordering (recommended)",
|
||||
"description": "\u2705 Both services receive the one AuthCache instance via constructor; the module export goes away (human: ~1 day / CC: ~20 min). \u2705 AuthCache exposes only get/put/invalidate keyed by a typed TenantKey and serializes writes per key, so mint-vs-revoke ordering is defined and testable. \u274c Every call site that imported the global must be threaded through the composition root."
|
||||
},
|
||||
{
|
||||
"label": "1B) Keep the module export; make SessionMint read-only",
|
||||
"description": "\u2705 Smaller change: only AuthBroker writes, so no two-writer race exists (human: ~2 hrs / CC: ~5 min). \u2705 Keeps the existing import shape for callers. \u274c Test isolation problem remains, and SessionMint still needs a write path for new sessions, which pushes the race into AuthBroker."
|
||||
},
|
||||
{
|
||||
"label": "1C) Proceed as written",
|
||||
"description": "\u2705 Zero re-planning; matches the current draft. \u2705 Fastest path to a first commit. \u274c Ships an auth cache with two unordered writers and shared test state on a multi-tenant path."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D4 \u2014 Issue 1 [P1] (confidence 9/10) PLAN.md:19-20: AuthBroker and SessionMint share a global mutable AuthCache via module-level export, and both mutate it.\nProject/branch/task: gstack-plan-count on main \u2014 Section 1 Architecture, Multi-tenant Auth Refactor.\nELI10: Two services write to the same cache through a global variable. The plan also says the cache 'does not serialize mutations' (PLAN.md:10). So writes can interleave: SessionMint stores a session while AuthBroker invalidates the same tenant on logout, and the last writer wins with no ordering. A module-level export also means tests share one instance, so one test's leftover token can make another pass or fail. Search check [Layer 1]: dependency injection is the standard cure; module singletons are the documented trap.\nStakes if we pick wrong: a revoked token stays valid for one tenant because a concurrent mint re-wrote it, and no test can reproduce it because the shared instance hides ordering.\nRecommendation: 1A because a single instance is fine, a global accessor is not. Inject it once from the composition root and give the facade a narrow write API that owns ordering. Maps to your 'explicit over clever' preference.\nCompleteness: A=10/10, B=6/10, C=2/10.\nNet: one extra constructor parameter per service buys deterministic writes and isolated tests.": "1A) Inject AuthCache; narrow write API with per-key ordering (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:13:26.078Z"
|
||||
}
|
||||
}
|
||||
+367
@@ -0,0 +1,367 @@
|
||||
# Plan: Multi-tenant Auth Refactor (eng-reviewed)
|
||||
|
||||
Reviewed by `/plan-eng-review` on 2026-09-10, branch `main`, commit `18d2903`.
|
||||
Source plan: `PLAN.md`. Mode: SCOPE_REDUCED (Step 0, decision D2).
|
||||
Every remedy below was approved individually (D1-D14). Nothing was auto-decided.
|
||||
|
||||
## Context
|
||||
|
||||
The repo's auth path is single-flow (`legacyAuthFlow()`) over a cache adapter that
|
||||
already keys entries by tenant ID, issuer, audience, and policy version, evicts expired
|
||||
tokens, and invalidates on logout, token revocation, and tenant suspension. This work
|
||||
introduces a broker/mint split so tenants can be served by a dedicated broker path
|
||||
while the adapter and its invalidation hooks stay untouched.
|
||||
|
||||
The original draft (PLAN.md) named its own smells but did not resolve them: a global
|
||||
mutable cache shared by two writers, a 60-line validator that swallows three error
|
||||
classes, a big-bang rewrite of the live login path with no regression test, five
|
||||
sequential IDP round trips, and five new units across 12 files. This reviewed plan
|
||||
resolves each one with a specific, approved remedy.
|
||||
|
||||
Goal (stated here because the draft had none): tenant-scoped authentication with the
|
||||
same user-visible behavior as today, no cross-tenant token reuse, fail-closed error
|
||||
handling, and login latency bounded by one IDP round trip instead of five.
|
||||
|
||||
## Existing contracts retained (unchanged from draft)
|
||||
|
||||
The existing cache adapter keys entries by tenant ID, issuer, audience, and policy
|
||||
version. It evicts expired tokens and invalidates entries on logout, token revocation,
|
||||
or tenant suspension. The adapter, its invalidation hooks, and their existing tests
|
||||
remain in use unchanged. `AuthCache` is a service-facing facade over that one adapter,
|
||||
with one backing cache. The adapter does not serialize mutations; the facade now does
|
||||
(decision 1A).
|
||||
|
||||
## Scope (reduced per D2)
|
||||
|
||||
| Unit | Status | Reason |
|
||||
|------|--------|--------|
|
||||
| `AuthBroker` | NEW service | Does new work: tenant-routed authentication |
|
||||
| `SessionMint` | NEW service | Does new work: session issuance |
|
||||
| `AuthCache` | NEW facade | Single service-facing wrapper over the existing adapter |
|
||||
| `TokenStore` | CUT (merged into `AuthCache`) | Second wrapper over the same adapter; duplication |
|
||||
| `RequestPolicy` | CUT as a class; becomes a typed value | Policy version is already a key dimension of the adapter |
|
||||
| `validateAndDispatch()` | SPLIT into `validateToken()` + `dispatch()` | Decision 5A |
|
||||
| `legacyAuthFlow()` | KEPT behind a flag until parity | Decision 2A |
|
||||
|
||||
Net: 3 new units, roughly 7-8 files (draft: 5 units, 12 files).
|
||||
|
||||
## Architecture
|
||||
|
||||
### Request flow (ASCII; also lives as a comment atop `auth/AuthBroker` per 4A)
|
||||
|
||||
```
|
||||
request(tenantId, token)
|
||||
│
|
||||
▼
|
||||
┌──────────────────┐ flag OFF ┌──────────────────┐
|
||||
│ auth router │─────────────▶│ legacyAuthFlow() │──▶ response (unchanged)
|
||||
│ (per-tenant flag)│ └──────────────────┘
|
||||
└────────┬─────────┘
|
||||
│ flag ON
|
||||
▼
|
||||
┌──────────────────┐ TenantKey ┌──────────────────┐ miss ┌──────────┐
|
||||
│ AuthBroker │──────────────▶│ AuthCache facade │──────────▶│ IDP │
|
||||
│ .authenticate() │◀──────────────│ get/put/invalid. │◀──────────│ (∥ calls)│
|
||||
└────────┬─────────┘ hit/value │ per-key ordering │ settled └──────────┘
|
||||
│ │ + coalescing │
|
||||
│ validateToken() → Result└────────┬─────────┘
|
||||
│ Valid | Expired | Revoked | │ one instance, injected
|
||||
│ IssuerMismatch | IdpUnavailable │ from composition root
|
||||
▼ ▼
|
||||
┌──────────────────┐ ┌──────────────────┐
|
||||
│ dispatch(Result) │ │ SessionMint │
|
||||
│ → route / 401 / │ │ .mint(TenantKey) │
|
||||
│ 403 / 503 │ └──────────────────┘
|
||||
└──────────────────┘
|
||||
```
|
||||
|
||||
### Invalidation fan-in (ASCII; also a comment atop `auth/AuthCache`)
|
||||
|
||||
```
|
||||
logout ─────────┐
|
||||
token revoked ──┼──▶ existing adapter hooks ──▶ AuthCache.invalidate(TenantKey)
|
||||
tenant suspend ─┘ (unchanged) │ serialized per key with
|
||||
│ in-flight puts (1A)
|
||||
▼
|
||||
one backing cache
|
||||
keyed by TenantKey only (3A)
|
||||
```
|
||||
|
||||
### Decisions applied
|
||||
|
||||
1. **1A — Inject `AuthCache`; narrow write API with per-key ordering.** No module-level
|
||||
export. The composition root constructs one `AuthCache` and passes it to
|
||||
`AuthBroker` and `SessionMint` by constructor. The facade exposes `get`, `put`,
|
||||
`invalidate`, each keyed by `TenantKey`, and keeps a per-key in-flight map so a
|
||||
mint and an invalidate on the same key settle in a defined order.
|
||||
2. **2A — Strangler seam.** One entry point routes per tenant (config flag) to
|
||||
`legacyAuthFlow()` or `AuthBroker`. Rollback is a config flip. Legacy removal is a
|
||||
TODO with an explicit exit condition (see TODOS.md updates).
|
||||
3. **3A — Typed `TenantKey` built only inside `AuthCache`.** Record of
|
||||
`{ tenantId, issuer, audience, policyVersion }`; the facade is the only code that
|
||||
serializes it. A property test asserts distinct tenants with identical issuer and
|
||||
audience never collide.
|
||||
4. **4A — Diagrams and failure table in the plan and as code comments** in
|
||||
`AuthBroker` and `AuthCache`. Diagram maintenance is part of any later change.
|
||||
5. **8A — Per-key request coalescing** inside `AuthCache`, reusing the 1A in-flight
|
||||
map: concurrent misses for one key share one IDP fetch; a failed shared fetch rejects
|
||||
every waiter with the typed error.
|
||||
|
||||
### Production failure scenarios per new codepath
|
||||
|
||||
| Codepath | Realistic failure | Handled by | User sees |
|
||||
|----------|-------------------|------------|-----------|
|
||||
| router flag lookup | flag store unreachable | default to legacy path, log | normal login |
|
||||
| `AuthBroker.authenticate` | IDP unreachable | `IdpUnavailable` result, fail closed | 503 with retry hint |
|
||||
| `AuthCache.put` vs `invalidate` | interleaved writes on one key | per-key ordering (1A) | revoked stays revoked |
|
||||
| `AuthCache` key build | caller omits tenant | impossible: only facade builds key (3A) | n/a |
|
||||
| `AuthCache` miss burst | N concurrent misses, hot tenant | coalescing (8A) | one round trip |
|
||||
| `validateToken` ∥ IDP calls | 1 of 5 rejects or times out | per-call typed classification (7A) | 401 or 503, never hang |
|
||||
| `SessionMint.mint` | double submit | idempotent per (TenantKey, claims) | one session |
|
||||
| tenant suspended mid-request | suspension lands between get and dispatch | invalidate wins; dispatch re-checks | 403 |
|
||||
|
||||
## Code quality
|
||||
|
||||
- **5A — Split `validateAndDispatch()`.** `validateToken()` returns a typed
|
||||
`Result` (`Valid | Expired | Revoked | IssuerMismatch | IdpUnavailable`);
|
||||
`dispatch()` switches on it. One error boundary at the entry point logs with tenant
|
||||
context and maps to 401/403/503. No catch swallows anything; unknown errors fail
|
||||
closed. Callers that relied on silent fallthrough will start seeing 401s, which is the
|
||||
intended behavior change.
|
||||
- **DRY (resolved by D2):** `TokenStore` merged into `AuthCache`; one wrapper over one
|
||||
adapter.
|
||||
- **Consistency (resolved by D2):** the draft listed 4 new classes in one section and
|
||||
named a 5th (`AuthBroker`) in another; the scope table above is now the single list.
|
||||
- **Explicit over clever:** `TenantKey` is a record, not a concatenated string;
|
||||
`RequestPolicy` is a typed value passed into `TenantKey.policyVersion`, not a class.
|
||||
|
||||
## Performance
|
||||
|
||||
- **7A — Parallel IDP calls with per-call classification.** Use `Promise.allSettled`
|
||||
(or `Promise.all` with a per-call catch) and map each outcome to a typed `AuthError`
|
||||
so a failed introspection reads differently from a failed key fetch. Per-call timeout
|
||||
budget so no request hangs. Cache the issuer discovery document and JWKS per issuer
|
||||
with TTL and a rotation-triggered refresh; most validations then need 0-1 live calls.
|
||||
- **8A — Coalescing** (above) bounds refills to one per key per expiry.
|
||||
- Observability (D13, built in this PR): per-tenant IDP latency histogram, cache hit
|
||||
ratio, coalesced-miss count, 401/503 rates, tagged by path (`legacy|broker`), emitted
|
||||
at the `AuthCache` facade and `validateToken()` boundary using the project's existing
|
||||
metrics client. Cap tenant-tag cardinality.
|
||||
|
||||
## Tests
|
||||
|
||||
Test framework: none detectable in this fixture repo (only `PLAN.md` is committed).
|
||||
The plan's `Promise.all` reference implies a Node/TypeScript runner; confirm the real
|
||||
repo's runner and naming convention before creating files. Paths below use
|
||||
`tests/auth/*.test.ts` as the convention to match.
|
||||
|
||||
### CRITICAL — regression (mandatory, REGRESSION RULE)
|
||||
|
||||
`legacyAuthFlow()` is live behavior being changed with no covering test (PLAN.md:27-28).
|
||||
Before any rewrite: `tests/auth/legacyAuthFlow.characterization.test.ts` records current
|
||||
outputs (including quirks) for valid, expired, revoked, issuer-mismatch, IDP-unavailable,
|
||||
and tenant-suspended inputs. This suite runs against the legacy path now and moves to
|
||||
`AuthBroker` when the flag is removed (TODO 1).
|
||||
|
||||
### Coverage diagram (after 6A every GAP below becomes a named test)
|
||||
|
||||
```
|
||||
CODE PATHS USER FLOWS
|
||||
[+] auth/AuthBroker [+] Login, flag ON (new path)
|
||||
├── authenticate(req, tenantKey) ├── [GAP→E2E] valid token → session issued
|
||||
│ ├── [PLANNED ★★] success — PLAN.md:14 ├── [GAP→E2E] expired → 401 + re-auth prompt
|
||||
│ ├── [PLANNED ★★] error — PLAN.md:14 ├── [GAP] revoked mid-session → 401 next request
|
||||
│ ├── [GAP] IDP unreachable → IdpUnavailable └── [GAP] tenant suspended mid-request → 403
|
||||
│ ├── [GAP] cache hit / cache miss (both branches)
|
||||
│ └── [GAP] concurrent mint + revoke, same key [+] Login, flag OFF (legacy path)
|
||||
└── flag router (legacy | broker) ├── [REGRESSION][CRITICAL] characterization suite
|
||||
├── [GAP] flag on → AuthBroker └── [GAP→E2E] parity: same input, same result
|
||||
└── [GAP] flag off → legacyAuthFlow
|
||||
[+] auth/SessionMint [+] Error states
|
||||
└── mint(tenantKey, claims) ├── [GAP] IDP 5xx → 503, never a silent pass
|
||||
├── [PLANNED ★★] success/error — PLAN.md:14 ├── [GAP] IDP timeout → bounded wait, then 503
|
||||
└── [GAP] double-submit mint is idempotent └── [GAP] 1-of-5 IDP call fails → typed error
|
||||
[+] auth/AuthCache (facade over existing adapter)
|
||||
├── [GAP] TenantKey isolation property test (3A)
|
||||
├── [GAP] per-key write ordering, mint vs invalidate (1A)
|
||||
├── [GAP] coalesced miss: N waiters, 1 fetch; failed fetch rejects all (8A)
|
||||
├── [GAP] logout/revoke/suspend invalidation via facade
|
||||
└── [EXISTING ★★★] adapter eviction/invalidation — PLAN.md:12-13
|
||||
[+] auth/validate.ts
|
||||
├── validateToken(): [GAP] Valid/Expired/Revoked/IssuerMismatch/IdpUnavailable (5)
|
||||
├── validateToken(): [GAP] ∥ IDP calls all-ok / one-rejects / all-reject
|
||||
├── validateToken(): [GAP] JWKS cache hit / stale after rotation → refresh
|
||||
└── dispatch(): [GAP] each Result variant → route + status
|
||||
|
||||
COVERAGE (draft): 4/33 paths (12%) | Code: 4/23 | User flows: 0/10
|
||||
QUALITY: ★★★:1 ★★:3 | GAPS: 29 (3 E2E, 0 eval, 1 REGRESSION)
|
||||
TARGET (this plan): 33/33
|
||||
```
|
||||
|
||||
### Tests to write (6A)
|
||||
|
||||
| File | Asserts |
|
||||
|------|---------|
|
||||
| `tests/auth/legacyAuthFlow.characterization.test.ts` | CRITICAL: current behavior per input class, quirks included |
|
||||
| `tests/auth/authBroker.test.ts` | success; each `Result` variant; cache hit vs miss; IDP unreachable → `IdpUnavailable` |
|
||||
| `tests/auth/authRouter.test.ts` | flag on → broker; flag off → legacy; flag store down → legacy + log |
|
||||
| `tests/auth/sessionMint.test.ts` | success/error; double submit yields one session |
|
||||
| `tests/auth/authCache.property.test.ts` | property: distinct tenants, same issuer+audience, never collide; suspension invalidates only that tenant |
|
||||
| `tests/auth/authCache.concurrency.test.ts` | mint then invalidate on one key settles in order; N concurrent misses → 1 fetch; failed fetch rejects all waiters |
|
||||
| `tests/auth/authCache.invalidation.test.ts` | logout/revoke/suspend hooks reach the facade and clear the right key |
|
||||
| `tests/auth/validateToken.test.ts` | 5 Result variants; ∥ IDP all-ok / one-rejects / all-reject; per-call timeout; JWKS cache hit and rotation refresh |
|
||||
| `tests/auth/dispatch.test.ts` | each Result → route or 401/403/503; nothing swallowed |
|
||||
| `tests/auth/e2e/login.e2e.test.ts` [→E2E] | valid login → authed request → logout → 401 (stubbed IDP) |
|
||||
| `tests/auth/e2e/expired.e2e.test.ts` [→E2E] | expired → 401 → re-auth → works |
|
||||
| `tests/auth/e2e/parity.e2e.test.ts` [→E2E] | same inputs through legacy and broker produce identical outcomes |
|
||||
|
||||
### QA test plan artifact
|
||||
|
||||
Written for `/qa` and `/qa-only` at
|
||||
`~/.gstack/projects/gstack-plan-count-4WfoHa/vercel-sandbox-main-eng-review-test-plan-20260910-181559.md`.
|
||||
|
||||
## What already exists
|
||||
|
||||
| Sub-problem | Existing code | Plan's use |
|
||||
|-------------|---------------|------------|
|
||||
| Tenant/issuer/audience/policy keying | cache adapter | Reused via `AuthCache`; `TenantKey` formalizes it |
|
||||
| Expiry eviction | cache adapter | Reused unchanged |
|
||||
| Invalidation on logout/revoke/suspend | adapter hooks + tests | Reused unchanged; facade routes through them |
|
||||
| Login behavior | `legacyAuthFlow()` | Kept behind flag; characterized; removed later |
|
||||
| Token storage | cache adapter | Draft rebuilt it as `TokenStore`; now cut |
|
||||
| Policy versioning | adapter key dimension | Draft rebuilt it as `RequestPolicy` class; now a typed value |
|
||||
| Metrics emission | project metrics client (assumed) | Reuse; do not add a client |
|
||||
|
||||
## NOT in scope
|
||||
|
||||
- **Removing `legacyAuthFlow()` and the flag** — TODO 1; needs parity data first.
|
||||
- **Load-testing the miss storm / IDP rate limits** — TODO 3; separate harness.
|
||||
- **Changing the cache adapter or its invalidation hooks** — retained contract.
|
||||
- **New IDP client library** — reuse the existing one; only call shape changes (7A).
|
||||
- **`TokenStore` and `RequestPolicy` classes** — cut in D2, not deferred.
|
||||
- **Distribution/CI changes** — no new artifact type is introduced.
|
||||
|
||||
## TODOS.md updates (create `TODOS.md` at implementation; file does not exist yet)
|
||||
|
||||
```markdown
|
||||
# TODOS
|
||||
|
||||
## Auth
|
||||
|
||||
### Remove legacyAuthFlow() and the routing flag
|
||||
**What:** Delete legacyAuthFlow, the per-tenant flag, and the router branch; point the characterization suite at AuthBroker.
|
||||
**Why:** Finish the strangler; one login path, no dual-path drift, no flag config to audit.
|
||||
**Context:** Added by /plan-eng-review 2026-09-10 (decision 2A/D12). Exit condition: parity E2E green in staging and all tenants on the broker path for two release cycles. Start at auth router.
|
||||
**Effort:** S **Priority:** P2 **Depends on:** 2A shipped; parity.e2e green; all tenants flagged on.
|
||||
|
||||
### Load-test cache-miss storm and IDP rate-limit behavior
|
||||
**What:** k6/artillery scenario against a stubbed, call-counting IDP; assert outbound calls per key per expiry == 1 and p99 login within budget.
|
||||
**Why:** Decision 8A (coalescing) rests on a medium-confidence bet about hot-tenant concurrency; this measures it before production does.
|
||||
**Context:** Added by /plan-eng-review 2026-09-10 (D14). Run before the first production policy-version bump. Read counts from the metrics added in this PR.
|
||||
**Effort:** M **Priority:** P2 **Depends on:** 7A, 8A, metrics (D13) merged.
|
||||
```
|
||||
|
||||
TODO 2 (observability) was chosen as "build now" (D13) and is in scope above.
|
||||
|
||||
## Worktree parallelization strategy
|
||||
|
||||
| Step | Modules touched | Depends on |
|
||||
|------|-----------------|------------|
|
||||
| T5 characterization suite | tests/auth/ (legacy) | — |
|
||||
| T2 AuthCache facade (1A, 3A, 8A) | auth/cache/ | — |
|
||||
| T3 validate/dispatch split (5A) | auth/validate/ | — |
|
||||
| T1 composition root + services | auth/broker/, auth/mint/, app bootstrap | T2 |
|
||||
| T4 flag-routed seam (2A) | auth/router/ | T1, T5 |
|
||||
| T7 ∥ IDP calls + JWKS cache (7A) | auth/idp/ | T3 |
|
||||
| T8 metrics (D13) | auth/cache/, auth/validate/ | T2, T3 |
|
||||
| T9 diagrams as comments (4A) | auth/broker/, auth/cache/ | T1, T2 |
|
||||
| T6 full test suite (6A) | tests/auth/ | T1-T4, T7 |
|
||||
|
||||
- **Lane A:** T2 → T1 → T4 (sequential, shared auth/cache → broker → router)
|
||||
- **Lane B:** T3 → T7 (sequential, shared auth/validate → auth/idp)
|
||||
- **Lane C:** T5 (independent)
|
||||
- **Execution:** launch A, B, C in parallel worktrees. Merge all three. Then T8 + T9
|
||||
(touch both lanes' modules). Then T6.
|
||||
- **Conflict flag:** T8 touches auth/cache/ (Lane A) and auth/validate/ (Lane B).
|
||||
Run it only after both lanes merge.
|
||||
|
||||
## Implementation Tasks
|
||||
Synthesized from this review's findings. Each task derives from a specific
|
||||
finding above. Run with Claude Code or Codex; checkbox as you ship.
|
||||
|
||||
- [ ] **T1 (P1, human: ~1 day / CC: ~20 min)** — auth/broker, auth/mint, bootstrap — Inject one `AuthCache` from the composition root; delete the module-level export
|
||||
- Surfaced by: Architecture — Issue 1 (PLAN.md:19-20), decision 1A
|
||||
- Files: auth/broker/AuthBroker.ts, auth/mint/SessionMint.ts, app/bootstrap.ts
|
||||
- Verify: `tests/auth/authBroker.test.ts` constructs services with a fake cache; no `import { authCache }` anywhere
|
||||
- [ ] **T2 (P1, human: ~1 day / CC: ~30 min)** — auth/cache — `AuthCache` facade: `TenantKey` record built only here, per-key write ordering, per-key request coalescing
|
||||
- Surfaced by: Architecture — Issues 1, 3 (PLAN.md:10-12, 19-20); Performance — Issue 8; decisions 1A, 3A, 8A
|
||||
- Files: auth/cache/AuthCache.ts, auth/cache/TenantKey.ts
|
||||
- Verify: `authCache.property.test.ts`, `authCache.concurrency.test.ts` green
|
||||
- [ ] **T3 (P1, human: ~1 day / CC: ~20 min)** — auth/validate — Split `validateAndDispatch()` into `validateToken()` returning typed `Result` and `dispatch()`; single fail-closed error boundary
|
||||
- Surfaced by: Code Quality — Issue 5 (PLAN.md:23-24), decision 5A
|
||||
- Files: auth/validate/validateToken.ts, auth/validate/dispatch.ts, auth/errors.ts
|
||||
- Verify: `validateToken.test.ts`, `dispatch.test.ts`; grep shows zero empty catch blocks
|
||||
- [ ] **T4 (P1, human: ~1 day / CC: ~20 min)** — auth/router — Per-tenant flag routes to `legacyAuthFlow()` or `AuthBroker`; flag-store failure defaults to legacy
|
||||
- Surfaced by: Architecture — Issue 2 (PLAN.md:27-28), decision 2A
|
||||
- Files: auth/router/authRouter.ts, config/flags
|
||||
- Verify: `authRouter.test.ts`; flipping the flag in staging switches paths without deploy
|
||||
- [ ] **T5 (P1, human: ~4 hrs / CC: ~15 min)** — tests/auth — CRITICAL regression: characterization suite for `legacyAuthFlow()` current behavior
|
||||
- Surfaced by: Tests — REGRESSION RULE (PLAN.md:27-28)
|
||||
- Files: tests/auth/legacyAuthFlow.characterization.test.ts
|
||||
- Verify: suite green against unmodified legacy before any other task merges
|
||||
- [ ] **T6 (P1, human: ~3 days / CC: ~1 hr)** — tests/auth — Close all 29 coverage gaps: unit, property, concurrency, and 3 E2E flows against a stubbed IDP
|
||||
- Surfaced by: Tests — Issue 6 (PLAN.md:14-16), decision 6A
|
||||
- Files: tests/auth/*.test.ts, tests/auth/e2e/*.e2e.test.ts (table above)
|
||||
- Verify: coverage report shows every diagram branch exercised; E2E suite green
|
||||
- [ ] **T7 (P2, human: ~1 day / CC: ~20 min)** — auth/idp — Parallel IDP calls with per-call typed classification, per-call timeout, issuer discovery + JWKS cache with TTL and rotation refresh
|
||||
- Surfaced by: Performance — Issue 7 (PLAN.md:31-32), decision 7A
|
||||
- Files: auth/idp/idpClient.ts, auth/idp/jwksCache.ts
|
||||
- Verify: `validateToken.test.ts` ∥ cases; p50 login latency ≈ 1 IDP round trip in staging
|
||||
- [ ] **T8 (P2, human: ~3 hrs / CC: ~10 min)** — auth/cache, auth/validate — Per-tenant metrics: IDP latency, hit ratio, coalesced misses, 401/503 rates, tagged by path
|
||||
- Surfaced by: TODO 2 chosen "build now" (D13)
|
||||
- Files: auth/cache/AuthCache.ts, auth/validate/validateToken.ts, using existing metrics client
|
||||
- Verify: metrics visible in staging dashboard per tenant and path; cardinality cap enforced
|
||||
- [ ] **T9 (P2, human: ~1 hr / CC: ~5 min)** — auth/broker, auth/cache — Embed the request-flow and invalidation ASCII diagrams as file-header comments
|
||||
- Surfaced by: Architecture — Issue 4, decision 4A
|
||||
- Files: auth/broker/AuthBroker.ts, auth/cache/AuthCache.ts
|
||||
- Verify: diagrams match the plan; reviewed on any later change to those files
|
||||
- [ ] **T10 (P3, human: ~10 min / CC: ~1 min)** — TODOS.md — Create file with the two entries above
|
||||
- Surfaced by: TODOS.md updates (D12, D14)
|
||||
- Files: TODOS.md
|
||||
- Verify: entries follow the gstack TODOS format
|
||||
|
||||
## Suppressed findings
|
||||
|
||||
None. Every finding scored 6/10 or higher. Issue 8 (miss storm) was reported at 6/10
|
||||
with the medium-confidence caveat and accepted with a load-test TODO to measure it.
|
||||
|
||||
## Completion summary
|
||||
|
||||
- Step 0: Scope Challenge — scope reduced per recommendation (5 units/12 files → 3 units/~7-8 files)
|
||||
- Architecture Review: 4 issues found, 4 resolved (all complete option)
|
||||
- Code Quality Review: 1 issue found, 1 resolved (2 more resolved by Step 0)
|
||||
- Test Review: diagram produced, 29 gaps identified (1 CRITICAL regression), all added to plan
|
||||
- Performance Review: 2 issues found, 2 resolved
|
||||
- NOT in scope: written
|
||||
- What already exists: written
|
||||
- TODOS.md updates: 3 items proposed (2 added, 1 built now)
|
||||
- Failure modes: 3 critical gaps flagged in the draft (unordered two-writer cache, swallowed auth errors, hand-built tenant keys); 0 remain after approved remedies
|
||||
- Outside voice: skipped (codex_reviews disabled; no native fallback by design)
|
||||
- Parallelization: 3 lanes, 3 parallel then 2 sequential steps
|
||||
- Lake Score: 8/8 recommendations chose the complete option
|
||||
|
||||
## GSTACK REVIEW REPORT
|
||||
|
||||
| Review | Trigger | Why | Runs | Status | Findings |
|
||||
|--------|---------|-----|------|--------|----------|
|
||||
| CEO Review | `/plan-ceo-review` | Scope & strategy | 0 | — | — |
|
||||
| Outside Review | codex via `/plan-eng-review` | Independent 2nd opinion | 1 | disabled | skipped (codex_reviews disabled) |
|
||||
| Eng Review | `/plan-eng-review` | Architecture & tests (required) | 1 | clean (PLAN) | 8 issues + 29 test gaps, 0 critical gaps remaining, mode SCOPE_REDUCED |
|
||||
| Design Review | `/plan-design-review` | UI/UX gaps | 0 | — | — |
|
||||
| DX Review | `/plan-devex-review` | Developer experience gaps | 0 | — | — |
|
||||
|
||||
**OUTSIDE COVERAGE:** provider codex, phase plan-review, outside_status disabled (user config `codex_reviews=disabled`); no outside findings; no native subagent fallback was dispatched because disabled is an opt-out, not a provider failure.
|
||||
|
||||
**VERDICT:** ENG CLEARED — ready to implement. CEO and Design reviews not run (backend-only auth refactor; neither is required).
|
||||
|
||||
NO UNRESOLVED DECISIONS
|
||||
+385
@@ -0,0 +1,385 @@
|
||||
# Plan: Multi-tenant Auth Refactor (reviewed)
|
||||
|
||||
Reviewed by `/plan-eng-review` on 2026-09-10 against PLAN.md at commit 0d7f121.
|
||||
Nine decisions (D1-D9) were made interactively; each is recorded inline where it
|
||||
changes the plan. Original plan text is kept where it still stands and marked
|
||||
**(revised)** where a decision changed it.
|
||||
|
||||
## Context
|
||||
|
||||
The auth path is being refactored for multi-tenancy. The original plan introduced
|
||||
two services (`AuthBroker`, `SessionMint`) sharing a mutable module-level
|
||||
`AuthCache`, added `TokenStore` and `RequestPolicy` classes, rewrote
|
||||
`legacyAuthFlow()` in place, and parallelized five IDP calls. Review found the
|
||||
shape was over-built relative to the existing cache adapter, had a fail-open race
|
||||
between two cache writers, swallowed errors in the dispatcher, had no rollout or
|
||||
rollback path, and no regression protection for the flow being rewritten.
|
||||
|
||||
The outcome after review: a thinner auth path (two new services, no new cache
|
||||
layer), a single cache writer with versioned writes so revocation always wins,
|
||||
a fail-closed dispatcher with typed errors, a per-tenant strangler-fig rollout
|
||||
with a loud fallback, full unit + integration coverage, and faster logins with
|
||||
fewer IDP calls.
|
||||
|
||||
## Existing contracts retained
|
||||
|
||||
The existing cache adapter keys entries by tenant ID, issuer, audience, and
|
||||
policy version. It evicts expired tokens and invalidates entries on logout,
|
||||
token revocation, or tenant suspension. Those validity and tenant-key rules are
|
||||
unchanged. The adapter, its invalidation hooks, and their existing tests remain
|
||||
in use.
|
||||
|
||||
**(revised, D2)** One narrow addition to the adapter: the write path accepts a
|
||||
policy-version tag and rejects a write whose tag is older than the current
|
||||
version for that tenant. Existing adapter tests stay green; new tests cover the
|
||||
rejection.
|
||||
|
||||
**(revised, D1)** `AuthCache` and `TokenStore` are not built. The adapter is the
|
||||
single cache layer and is passed to services by constructor injection.
|
||||
|
||||
## Architecture (revised, D1 + D2)
|
||||
|
||||
New types: `AuthBroker` (read path) and `SessionMint` (sole write path).
|
||||
`RequestPolicy` is a pure function over request + tenant config, not a class.
|
||||
No module-level exports of mutable state; both services receive the adapter,
|
||||
the IDP client, and the flag reader in their constructors.
|
||||
|
||||
```
|
||||
per-tenant flag (D3)
|
||||
│
|
||||
request ──> router ────────┼──────────────> legacyAuthFlow() (unflagged tenants,
|
||||
│ flag-store-down fallback D9)
|
||||
└──────────────> validateAndDispatch() (flattened, D4)
|
||||
│
|
||||
┌────────────────────────┴──────────────────────┐
|
||||
▼ ▼
|
||||
AuthBroker (READ only) SessionMint (SOLE WRITER)
|
||||
│ get(cacheKeyFor(...)) │ put(key, entry, policyVersion)
|
||||
▼ ▼
|
||||
┌──────────────────── existing cache adapter ──────────────────────┐
|
||||
│ keys: tenant|issuer|audience|policyVersion │
|
||||
│ rejects put() whose policyVersion < current (D2) │
|
||||
│ invalidates on logout / revocation / suspension (unchanged) │
|
||||
└───────────────────────────────────────────────────────────────────┘
|
||||
▲
|
||||
SessionMint ──> IDP client (static cache + parallel + timeout, D7)
|
||||
```
|
||||
|
||||
### Cache write ownership (D2)
|
||||
|
||||
Only `SessionMint` writes. `AuthBroker` reads and, on revocation or suspension
|
||||
signals, calls the adapter's existing invalidation hooks. Every write carries
|
||||
the policy version read at validation start via `readPolicyVersion()`. The
|
||||
adapter rejects a write whose version is stale and the rejection is logged at
|
||||
warn with tenant ID and key. This makes "revoke racing a mint resurrects the
|
||||
token" structurally impossible rather than unlikely.
|
||||
|
||||
```
|
||||
time ──────────────────────────────────────────────────────────────>
|
||||
SessionMint: read pv=7 ─── validate (IDP) ─────────── put(key, e, pv=7) ✗ rejected, logged
|
||||
Admin: revoke ──> invalidate(key), pv := 8
|
||||
AuthBroker: get(key) → miss → deny ✓
|
||||
```
|
||||
|
||||
### Rollout (D3 + D9)
|
||||
|
||||
`legacyAuthFlow()` stays callable. A per-tenant flag routes each tenant to the
|
||||
new flow or legacy. Rollout is tenant by tenant, starting with an internal
|
||||
tenant. Rollback is a flag flip.
|
||||
|
||||
Flag-store failure: lookup has a short timeout; on timeout or error the router
|
||||
falls back to `legacyAuthFlow()`, emits a warn-level structured log and a
|
||||
`auth.flag_fallback` metric, and an integration test stubs the flag store as
|
||||
down. Legacy removal is tracked in TODOS.md (D8) with the exit condition
|
||||
"all tenants flagged on, no flips for one full release cycle".
|
||||
|
||||
### Security architecture
|
||||
|
||||
* Tenant isolation boundary is the cache key. One builder (D5) makes the tenant
|
||||
field structurally required.
|
||||
* Every error class in the dispatcher denies (D4). Unknown errors deny.
|
||||
* Revocation always wins the race (D2).
|
||||
* Flag-store outage degrades to the known-good legacy path, never to an
|
||||
unvalidated pass (D9).
|
||||
|
||||
### Production failure scenarios per new codepath
|
||||
|
||||
| Codepath | Realistic failure | Test | Handling | User sees |
|
||||
|---|---|---|---|---|
|
||||
| `validateAndDispatch()` boundary | Unexpected exception mid-validation | yes (T4/T6) | deny + structured log (D4) | clear 401/403 |
|
||||
| `AuthBroker` read | Adapter throws / cache backend down | yes (T6) | deny, log, metric | clear 503 |
|
||||
| `SessionMint` write | Stale policy version after revoke | yes (T2/T6) | write rejected + warn log (D2) | denied on next request |
|
||||
| `SessionMint` write | Adapter write throws | yes (T6) | deny, no partial state | clear 503 |
|
||||
| IDP client | One of N calls rejects | yes (T7) | Promise.all rejects, all outcomes logged, deny | clear 503 |
|
||||
| IDP client | Call hangs | yes (T7) | per-call timeout → deny | clear 503, retry safe |
|
||||
| IDP static cache | JWKS key rotation, unknown `kid` | yes (T7) | one forced refetch, then deny | brief retry, then works |
|
||||
| Flag router | Flag store unreachable | yes (T3/T6) | legacy fallback + metric (D9) | nothing, legacy behavior |
|
||||
| `cacheKeyFor()` | Two tenants collide | yes (T5) | impossible by construction | n/a |
|
||||
|
||||
**Critical gaps (no test, no handling, silent): 0.**
|
||||
|
||||
## Code quality (revised, D4 + D5)
|
||||
|
||||
`validateAndDispatch()` was 60 lines with three nested try/catch blocks, each
|
||||
swallowing a different error class. It is rewritten as a linear pipeline of
|
||||
small steps:
|
||||
|
||||
```
|
||||
parseToken ──> resolveTenant ──> checkPolicy ──> validateWithIdp ──> mintSession ──> dispatch
|
||||
│ │ │ │ │
|
||||
TokenError TenantError PolicyError IdpError CacheError
|
||||
└───────────────┴────────────────┴─────────────────┴──────────────────┘
|
||||
│
|
||||
single boundary catch:
|
||||
map error → explicit DENY result
|
||||
structured log {tenant, step, errorClass}
|
||||
metric auth.deny{reason}
|
||||
unknown Error → DENY (fail closed)
|
||||
```
|
||||
|
||||
Typed errors: `TokenError`, `TenantError`, `PolicyError`, `IdpError`,
|
||||
`CacheError`, all extending `AuthError`. Callers that relied on a swallowed
|
||||
error to continue now receive an explicit deny and are updated in this PR.
|
||||
|
||||
DRY: `cacheKeyFor(tenantId, issuer, audience, policyVersion)` and
|
||||
`readPolicyVersion(tenantId)` live in one shared module used by `AuthBroker`
|
||||
and `SessionMint`. One test asserts every field is present and ordered and that
|
||||
two different tenants never produce the same key.
|
||||
|
||||
Inline ASCII diagram comments to add at implementation:
|
||||
* adapter write path: the versioned-write timeline above
|
||||
* `validateAndDispatch()`: the pipeline diagram above
|
||||
* router: flag decision tree including the fallback branch
|
||||
* `SessionMint`: mint pipeline and the sole-writer contract
|
||||
|
||||
## Tests (revised, D6 + REGRESSION RULE)
|
||||
|
||||
**CRITICAL regression suite (mandatory, IRON RULE).** `legacyAuthFlow()` is
|
||||
existing behavior being rewritten and the original plan had no regression
|
||||
coverage. Before any rewrite, write a characterization suite that pins current
|
||||
behavior: valid token → allow, expired → deny, wrong issuer/audience → deny,
|
||||
suspended tenant → deny, logout invalidates. The suite runs against both the
|
||||
legacy path and the new flow (via the flag) for the whole rollout window.
|
||||
|
||||
Coverage target: every branch in the diagram below, unit and integration.
|
||||
|
||||
```
|
||||
CODE PATHS USER FLOWS
|
||||
[~] legacyAuthFlow() (flag-routed, kept) [+] Login / token validation
|
||||
├── [CRITICAL] regression: valid token → allow ├── [→E2E] Login on flagged tenant → new flow
|
||||
├── [CRITICAL] regression: expired → deny ├── [→E2E] Login on unflagged tenant → legacy
|
||||
├── [CRITICAL] regression: wrong audience/issuer → deny ├── [→E2E] Flag flipped mid-session → no lockout
|
||||
└── [CRITICAL] regression: suspended tenant → deny └── [→E2E] Flag store down → legacy + metric (D9)
|
||||
[+] validateAndDispatch() (flattened, D4) [+] Revocation / logout
|
||||
├── happy path → dispatch ├── [→E2E] Revoke → next request denied
|
||||
├── TokenError → deny + log ├── [→E2E] Revoke racing mint → stale write rejected (D2)
|
||||
├── TenantError → deny + log └── Tenant suspended → all tokens denied
|
||||
├── PolicyError → deny + log
|
||||
├── IdpError → deny + log [+] Tenant isolation
|
||||
├── CacheError → deny + log ├── [→E2E] Tenant A token never validates for B
|
||||
└── unknown Error → deny + log (fail closed) └── Same issuer/audience, different tenant → miss
|
||||
[+] AuthBroker (read path)
|
||||
├── cache hit → allow [+] Error states
|
||||
├── cache miss → IDP validate ├── IDP timeout → clear 503, not 401
|
||||
└── adapter throws → deny ├── IDP 5xx → clear 503, retry safe
|
||||
[+] SessionMint (sole writer, D2) └── Partial IDP failure → deny, all outcomes logged
|
||||
├── mint → write with policy version
|
||||
├── stale version → write rejected + logged
|
||||
└── adapter write throws → deny, no partial state
|
||||
[+] cacheKeyFor() / readPolicyVersion() (D5)
|
||||
├── all four fields required and ordered
|
||||
└── two tenants never collide
|
||||
[+] IDP client (D7)
|
||||
├── static responses served from cache within TTL
|
||||
├── unknown kid → one forced JWKS refetch
|
||||
├── all parallel calls succeed
|
||||
├── one rejects → aggregate error, no hang
|
||||
└── per-call timeout fires
|
||||
|
||||
TARGET: 34/34 paths tested (100%) | Code paths: 22/22 | User flows: 12/12
|
||||
GAPS BEFORE REVIEW: 27 (7 E2E, 4 CRITICAL regression) | GAPS AFTER PLAN: 0
|
||||
```
|
||||
|
||||
Test harness: a fake adapter with an injectable write delay (for the D2 race
|
||||
test) and an IDP stub that can fail, hang, or rotate keys per call. Integration
|
||||
flows run against those fakes; no live IDP in CI.
|
||||
|
||||
## Performance (revised, D7)
|
||||
|
||||
Token validation issued 5 sequential IDP calls. Revised:
|
||||
|
||||
1. Static IDP responses (OIDC discovery document, JWKS) are cached with a TTL in
|
||||
the existing adapter; unknown `kid` triggers one forced refetch.
|
||||
Assumption: two of the five calls are these static fetches. If none are,
|
||||
step 2 still applies.
|
||||
2. Remaining calls run with `Promise.all`, each wrapped in a per-call timeout.
|
||||
Any rejection denies the request; all outcomes are logged so a partial
|
||||
failure is diagnosable.
|
||||
3. Expected result: login latency drops from 5 round trips to 1, IDP request
|
||||
volume drops by up to 40 percent, and a hung IDP call cannot hang a login.
|
||||
|
||||
## Scope (revised, D1)
|
||||
|
||||
Complexity check triggered on the original plan (12 files, 4 new classes plus
|
||||
`AuthBroker`). Reduced to: 2 new service types, 1 shared helper module, 1 typed
|
||||
error module, 1 narrow adapter change, 1 router change, the rewritten
|
||||
dispatcher, and tests. Roughly 8 source files plus tests.
|
||||
|
||||
## What already exists
|
||||
|
||||
| Sub-problem | Existing code | Plan now |
|
||||
|---|---|---|
|
||||
| Tenant-scoped cache keying, expiry, invalidation | cache adapter + hooks + tests | reused unchanged, one write-path addition (D2) |
|
||||
| Current auth behavior | `legacyAuthFlow()` | kept as flag fallback and regression oracle (D3) |
|
||||
| Service-facing cache facade | none needed; adapter API suffices | `AuthCache` dropped (D1) |
|
||||
| Token storage | adapter already stores tokens | `TokenStore` dropped (D1) |
|
||||
| Request policy evaluation | none; was a proposed class | pure function (D1) |
|
||||
|
||||
## NOT in scope
|
||||
|
||||
* **Deleting `legacyAuthFlow()` and the per-tenant flag** — tracked in
|
||||
TODOS.md (D8); happens after 100 percent rollout plus one release of bake.
|
||||
* **Adding an `AuthCache` facade** — only if a future consumer needs a narrower
|
||||
API than the adapter; not justified today (D1).
|
||||
* **Rewriting the cache adapter itself** — one write-path addition only (D2);
|
||||
its keying and invalidation rules are unchanged.
|
||||
* **IDP-side rate-limit negotiation or client-credential changes** — the
|
||||
static cache (D7) reduces load; anything beyond that is separate work.
|
||||
* **Multi-region cache consistency** — out of scope for this refactor; the
|
||||
versioned write (D2) is single-backing-cache correct as the plan states.
|
||||
* **New artifact distribution** — no new binary, package, or image; N/A.
|
||||
|
||||
## TODOS.md updates (apply at implementation start; plan mode forbade the edit)
|
||||
|
||||
```markdown
|
||||
# TODOS
|
||||
|
||||
## Auth
|
||||
|
||||
### Remove legacyAuthFlow() and the per-tenant new-flow flag
|
||||
|
||||
**What:** Delete legacyAuthFlow(), the per-tenant new-flow flag, the flag-store
|
||||
fallback path, and the legacy branch of the regression suite.
|
||||
|
||||
**Why:** Two auth code paths double the test and review cost of every future
|
||||
auth change and keep a fallback alive that no longer has anything to fall back
|
||||
from.
|
||||
|
||||
**Context:** /plan-eng-review D3 (2026-09-10) chose a strangler-fig rollout:
|
||||
legacyAuthFlow() stays callable behind a per-tenant flag while the new
|
||||
AuthBroker + SessionMint flow rolls out tenant by tenant. D9 added a
|
||||
flag-store-down fallback to legacy. The regression suite pins legacy behavior
|
||||
and runs against both paths during rollout. Start in the router and the
|
||||
regression suite; the flag reader and fallback metric go with them.
|
||||
|
||||
**Effort:** S
|
||||
**Priority:** P2
|
||||
**Depends on:** All tenants flagged on to the new flow with no flag flips for
|
||||
one full release cycle.
|
||||
```
|
||||
|
||||
## Worktree parallelization strategy
|
||||
|
||||
| Step | Modules touched | Depends on |
|
||||
|---|---|---|
|
||||
| S1 Regression suite for `legacyAuthFlow()` | tests/auth/legacy | — |
|
||||
| S2 Shared helpers + typed errors | auth/shared | — |
|
||||
| S3 Adapter versioned-write rejection | cache adapter module | S2 (policy version helper) |
|
||||
| S4 IDP client: static cache, parallel, timeout | auth/idp | — |
|
||||
| S5 `AuthBroker` + `SessionMint` | auth/services | S2, S3, S4 |
|
||||
| S6 Flattened dispatcher + flag router + fallback | auth/dispatch | S2, S5 |
|
||||
| S7 Integration flows | tests/auth/integration | S5, S6 |
|
||||
|
||||
Lanes:
|
||||
* Lane A: S1 (independent)
|
||||
* Lane B: S2 → S3 → S5 → S6 (sequential, shared auth/ services and adapter)
|
||||
* Lane C: S4 (independent)
|
||||
* Lane D: S7 (after B and C merge)
|
||||
|
||||
Execution order: launch A, B, C in parallel worktrees. Merge A first (it is
|
||||
pure tests and gates the rewrite). Merge C, then B. Then D.
|
||||
|
||||
Conflict flags: Lanes B and C both live under auth/; keep S4 confined to
|
||||
auth/idp and its own test file to avoid merge conflicts with auth/services.
|
||||
|
||||
## Implementation Tasks
|
||||
Synthesized from this review's findings. Each task derives from a specific
|
||||
finding above. Run with Claude Code or Codex; checkbox as you ship.
|
||||
|
||||
- [ ] **T1 (P1, human: ~1 day / CC: ~15min)** — tests/auth/legacy — Write the CRITICAL characterization suite for `legacyAuthFlow()` before any rewrite; run it against legacy and new flow
|
||||
- Surfaced by: Test review — REGRESSION RULE, PLAN.md:27-28 and :14-16
|
||||
- Files: tests/auth/legacy/*, router flag stub
|
||||
- Verify: suite green on legacy path before S5/S6 land; green on both paths after
|
||||
- [ ] **T2 (P1, human: ~1.5 days / CC: ~25min)** — cache adapter + auth/services — Single writer: only `SessionMint` writes; writes carry policy version; adapter rejects stale writes with warn log
|
||||
- Surfaced by: Architecture — D2, PLAN.md:10, 19-20
|
||||
- Files: cache adapter write path, auth/services/SessionMint, auth/services/AuthBroker
|
||||
- Verify: unit test for stale-write rejection; integration test "revoke racing mint → denied"
|
||||
- [ ] **T3 (P1, human: ~1.25 days / CC: ~23min)** — auth/dispatch — Per-tenant flag router with legacy fallback on flag-store failure, timeout, warn log, `auth.flag_fallback` metric
|
||||
- Surfaced by: Architecture — D3 + D9, PLAN.md:27-28
|
||||
- Files: auth/dispatch/router, flag reader interface
|
||||
- Verify: integration tests for flagged, unflagged, mid-session flip, flag store down
|
||||
- [ ] **T4 (P1, human: ~1 day / CC: ~20min)** — auth/dispatch — Flatten `validateAndDispatch()` into a linear pipeline with typed errors and one fail-closed boundary
|
||||
- Surfaced by: Code quality — D4, PLAN.md:23-24
|
||||
- Files: auth/dispatch/validateAndDispatch, auth/shared/errors
|
||||
- Verify: one unit test per error class plus unknown-error → deny; no catch without a deny
|
||||
- [ ] **T5 (P2, human: ~2h / CC: ~5min)** — auth/shared — `cacheKeyFor()` and `readPolicyVersion()` helpers used by both services, with ordering and no-collision tests
|
||||
- Surfaced by: Code quality — D5, PLAN.md:7-8
|
||||
- Files: auth/shared/cacheKey, tests
|
||||
- Verify: unit tests; grep shows no other key construction in auth/
|
||||
- [ ] **T6 (P1, human: ~2 days / CC: ~30min)** — tests/auth/integration — Integration flows: revocation race, tenant isolation, flag routing, IDP partial failure, flag store down, adapter failures
|
||||
- Surfaced by: Test review — D6 coverage diagram, 7 [→E2E] flows
|
||||
- Files: tests/auth/integration/*, fake adapter with write delay, IDP stub
|
||||
- Verify: all 12 user flows in the diagram green
|
||||
- [ ] **T7 (P2, human: ~1 day / CC: ~20min)** — auth/idp — TTL-cache static IDP responses, `Promise.all` with per-call timeout for the rest, forced JWKS refetch on unknown kid
|
||||
- Surfaced by: Performance — D7, PLAN.md:31-32
|
||||
- Files: auth/idp/client, tests
|
||||
- Verify: unit tests for cache hit, rotation refetch, one-rejects, timeout; latency measurement 5 RTT → 1 RTT
|
||||
- [ ] **T8 (P2, human: ~0.5 day / CC: ~10min)** — auth/services — Drop `AuthCache` and `TokenStore`; `RequestPolicy` as pure function; constructor injection for adapter, IDP client, flag reader
|
||||
- Surfaced by: Step 0 — D1, PLAN.md:19-20, 35-36
|
||||
- Files: auth/services/*, auth/shared/requestPolicy
|
||||
- Verify: no module-level mutable exports in auth/ (grep); services unit-testable with fakes
|
||||
- [ ] **T9 (P3, human: ~5min / CC: ~1min)** — TODOS.md — Add the legacy-removal TODO with its exit condition
|
||||
- Surfaced by: TODOS.md updates — D8
|
||||
- Files: TODOS.md
|
||||
- Verify: entry present in the format above
|
||||
|
||||
## Completion summary
|
||||
|
||||
- Step 0: Scope Challenge — scope reduced per recommendation (D1: 4-5 new classes → 2 services + helpers)
|
||||
- Architecture Review: 2 issues found (D2 cache write race, D3 rollout); 1 follow-on failure mode (D9)
|
||||
- Code Quality Review: 2 issues found (D4 swallowed errors, D5 key builder DRY)
|
||||
- Test Review: diagram produced, 27 gaps identified (4 CRITICAL regression, 7 E2E); all added to plan (D6)
|
||||
- Performance Review: 1 issue found (D7)
|
||||
- NOT in scope: written
|
||||
- What already exists: written
|
||||
- TODOS.md updates: 1 item proposed to user (D8, accepted)
|
||||
- Failure modes: 0 critical gaps flagged
|
||||
- Outside voice: skipped (codex_reviews disabled)
|
||||
- Parallelization: 4 lanes, 3 parallel / 1 sequential follow-on
|
||||
- Lake Score: 7/7 coverage-scored recommendations chose the complete option (D1, D8 were kind-only)
|
||||
|
||||
Setup prompts deferred this run (not approvals): CLAUDE.md routing rules (plan
|
||||
mode forbids the edit and commit), /office-hours design-doc offer (explicit
|
||||
review request), cross-project learnings toggle (learnings store empty).
|
||||
|
||||
## GSTACK REVIEW REPORT
|
||||
|
||||
| Review | Trigger | Why | Runs | Status | Findings |
|
||||
|--------|---------|-----|------|--------|----------|
|
||||
| CEO Review | `/plan-ceo-review` | Scope & strategy | 0 | — | — |
|
||||
| Outside Review | codex via `/plan-eng-review` | Independent 2nd opinion | 1 | disabled | skipped (codex_reviews=disabled) |
|
||||
| Eng Review | `/plan-eng-review` | Architecture & tests (required) | 1 | clean (PLAN) | 32 issues, 0 critical gaps |
|
||||
| Design Review | `/plan-design-review` | UI/UX gaps | 0 | — | — |
|
||||
| DX Review | `/plan-devex-review` | Developer experience gaps | 0 | — | — |
|
||||
|
||||
### Suppressed findings (confidence below 7, appendix only)
|
||||
|
||||
- (confidence: 5/10) PLAN.md:19 vs :35 — `AuthBroker` is a new service but is missing from the "4 new classes" list; the real count was 5. Informational; resolved by D1.
|
||||
- (confidence: 5/10) PLAN.md:31-32 — the five IDP calls are not named; the D7 static-cache step assumes two are OIDC discovery and JWKS. Verify at implementation.
|
||||
- (confidence: 4/10) PLAN.md:35 — `RequestPolicy` contents are undefined; treated as a pure function (D1). Revisit if it needs state.
|
||||
|
||||
**OUTSIDE COVERAGE:** provider codex, phase plan-review, status disabled by config (`codex_reviews=disabled`), no findings; no native fallback was dispatched because disabled is an intentional opt-out, not a provider failure.
|
||||
|
||||
**VERDICT:** ENG CLEARED — ready to implement (1 clean plan-eng-review run within 7 days, 0 unresolved, 0 critical gaps). Outside review disabled by config; CEO, Design, DX reviews not run and not required.
|
||||
|
||||
NO UNRESOLVED DECISIONS
|
||||
+299
@@ -0,0 +1,299 @@
|
||||
{
|
||||
"provenance": {
|
||||
"sourceHead": "f26d569e0345cb1131d9ca52d4a43965085c3468",
|
||||
"sourceObservationSha256": "b23910ab9f477af2028213b6877b646aab26f8a0cb64aa21c05a0d7acf107dad",
|
||||
"publicNativeProofSha256": "faf46cf9393433cc1e01b7f3eb1c80a52ea7bc5e72cfc59ba7bb698856aed091",
|
||||
"historicalOutcome": "no_review_questions; downstream three seed predicates and mandatory legacy regression missing",
|
||||
"paidOutcomeReclassified": false
|
||||
},
|
||||
"calls": [
|
||||
{
|
||||
"sessionId": "9e247dab-a089-4c51-ad6d-f067e2d4e5fe",
|
||||
"toolUseId": "toolu_01KfPwnoysgRFbyNyYbxucvR",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D1 — Reduce the new-class count before we review the rest?\nProject/branch/task: main — Multi-tenant Auth Refactor plan (PLAN.md), 12 files, 4-5 new classes.\nELI10: The plan invents a new cache wrapper (AuthCache) and a new token store on top of a cache adapter that already does tenant keying, expiry, and invalidation. Every extra class is another place a bug can hide in the code that decides who is logged in. Fewer moving parts means fewer places for a revoked token to slip through.\nStakes if we pick wrong: Too many layers and the auth path becomes hard to reason about and test; too few and we cram policy logic into services that should stay thin.\nRecommendation: 1A because the adapter already enforces every cache rule the plan lists, so AuthCache and TokenStore add indirection without behavior. Maps to your 'engineered enough' and right-sized-diff preferences.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\n1A) Reduce: keep AuthBroker + SessionMint, inject the existing adapter, RequestPolicy as a pure function/config, drop AuthCache and TokenStore as classes (human: ~1 day less / CC: ~10 min less) (recommended)\n ✅ Two new types instead of five; roughly 7-8 files touched instead of 12, smaller review surface\n ✅ No new global; each service is unit-testable with a fake adapter passed in\n ❌ If a real service-facing narrowing of the adapter API is needed later, you add the facade then\n1B) Keep all four classes but replace the module-level global with constructor injection (human: ~2 days / CC: ~20 min)\n ✅ Preserves the plan's intended layering if AuthCache is meant to narrow the adapter API\n ✅ Still removes the shared-mutable-global hazard, so tests stay isolated\n ❌ Facade and TokenStore still duplicate what the adapter already guarantees\n1C) Proceed exactly as written (human: ~2 days / CC: ~20 min)\n ✅ No re-planning; the author's structure stands untouched\n ✅ Fastest path to starting implementation today\n ❌ Keeps a mutable global shared by two writers in the auth path, the riskiest shape available\nNet: You are trading a thinner, more testable auth path against preserving layering the plan has not yet justified.",
|
||||
"header": "Scope",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "1A Reduce (recommended)",
|
||||
"description": "AuthBroker + SessionMint with injected adapter; RequestPolicy as pure fn; drop AuthCache/TokenStore classes."
|
||||
},
|
||||
{
|
||||
"label": "1B Keep 4 classes, add DI",
|
||||
"description": "Keep the layering but remove the module-level global in favor of constructor injection."
|
||||
},
|
||||
{
|
||||
"label": "1C Proceed as-is",
|
||||
"description": "Review the plan exactly as written, 12 files and 4 classes with the shared global."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D1 — Reduce the new-class count before we review the rest?\nProject/branch/task: main — Multi-tenant Auth Refactor plan (PLAN.md), 12 files, 4-5 new classes.\nELI10: The plan invents a new cache wrapper (AuthCache) and a new token store on top of a cache adapter that already does tenant keying, expiry, and invalidation. Every extra class is another place a bug can hide in the code that decides who is logged in. Fewer moving parts means fewer places for a revoked token to slip through.\nStakes if we pick wrong: Too many layers and the auth path becomes hard to reason about and test; too few and we cram policy logic into services that should stay thin.\nRecommendation: 1A because the adapter already enforces every cache rule the plan lists, so AuthCache and TokenStore add indirection without behavior. Maps to your 'engineered enough' and right-sized-diff preferences.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\n1A) Reduce: keep AuthBroker + SessionMint, inject the existing adapter, RequestPolicy as a pure function/config, drop AuthCache and TokenStore as classes (human: ~1 day less / CC: ~10 min less) (recommended)\n ✅ Two new types instead of five; roughly 7-8 files touched instead of 12, smaller review surface\n ✅ No new global; each service is unit-testable with a fake adapter passed in\n ❌ If a real service-facing narrowing of the adapter API is needed later, you add the facade then\n1B) Keep all four classes but replace the module-level global with constructor injection (human: ~2 days / CC: ~20 min)\n ✅ Preserves the plan's intended layering if AuthCache is meant to narrow the adapter API\n ✅ Still removes the shared-mutable-global hazard, so tests stay isolated\n ❌ Facade and TokenStore still duplicate what the adapter already guarantees\n1C) Proceed exactly as written (human: ~2 days / CC: ~20 min)\n ✅ No re-planning; the author's structure stands untouched\n ✅ Fastest path to starting implementation today\n ❌ Keeps a mutable global shared by two writers in the auth path, the riskiest shape available\nNet: You are trading a thinner, more testable auth path against preserving layering the plan has not yet justified.": "1A Reduce (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:24:29.278Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "9e247dab-a089-4c51-ad6d-f067e2d4e5fe",
|
||||
"toolUseId": "toolu_014xtoxbQBBjjSJV7QxaiS1N",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D2 — Who is allowed to write to the auth cache?\nProject/branch/task: main — Multi-tenant Auth Refactor, AuthBroker and SessionMint both mutating one backing cache (PLAN.md:10, 19-20).\nELI10: Two services write to the same cache and nothing orders their writes. If one service removes a revoked token while the other is mid-way through storing that same token, the store lands last and the revoked token comes back to life. Users would see a logged-out or revoked session keep working.\nStakes if we pick wrong: A revoked or suspended tenant's tokens stay valid until natural expiry, silently, with no log line to find it.\nRecommendation: 2A because one writer plus a version check makes the race structurally impossible instead of merely unlikely; explicit over clever.\nCompleteness: 2A=10/10, 2B=7/10, 2C=3/10\nPros / cons:\n2A) Single writer + versioned writes: only SessionMint writes; AuthBroker reads and calls the adapter's existing invalidation hooks; every write carries the policy version read at validation start and the adapter rejects writes whose version is stale (human: ~1.5 days / CC: ~25 min) (recommended)\n ✅ Revocation racing a mint can never resurrect a token; the stale write is rejected and logged\n ✅ Ownership is obvious from the code: one class writes, one class reads, testable with a fake adapter\n ❌ Needs a compare-and-set or version-tag on the adapter write path, a small adapter change\n2B) Single writer only: SessionMint writes, AuthBroker reads and invalidates, no version check (human: ~1 day / CC: ~15 min)\n ✅ Removes the two-writer coupling with no adapter change at all\n ✅ Simpler to explain and diagram than a version scheme\n ❌ Invalidate-then-write-back is still possible inside a single mint that started before the revocation\n2C) Keep both writers, document the race as accepted (human: ~0 / CC: ~0)\n ✅ Zero implementation cost right now\n ✅ Matches the plan as written\n ❌ Ships a known fail-open path in auth; the first incident is a security incident\nNet: You are trading a small adapter change for a guarantee that revocation always wins the race.",
|
||||
"header": "Cache writes",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "2A Single writer + version (recommended)",
|
||||
"description": "SessionMint writes with policy-version tag; adapter rejects stale writes; AuthBroker reads/invalidates."
|
||||
},
|
||||
{
|
||||
"label": "2B Single writer only",
|
||||
"description": "SessionMint writes, AuthBroker reads and invalidates. No version check."
|
||||
},
|
||||
{
|
||||
"label": "2C Accept the race",
|
||||
"description": "Keep both writers as planned and document the known race."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D2 — Who is allowed to write to the auth cache?\nProject/branch/task: main — Multi-tenant Auth Refactor, AuthBroker and SessionMint both mutating one backing cache (PLAN.md:10, 19-20).\nELI10: Two services write to the same cache and nothing orders their writes. If one service removes a revoked token while the other is mid-way through storing that same token, the store lands last and the revoked token comes back to life. Users would see a logged-out or revoked session keep working.\nStakes if we pick wrong: A revoked or suspended tenant's tokens stay valid until natural expiry, silently, with no log line to find it.\nRecommendation: 2A because one writer plus a version check makes the race structurally impossible instead of merely unlikely; explicit over clever.\nCompleteness: 2A=10/10, 2B=7/10, 2C=3/10\nPros / cons:\n2A) Single writer + versioned writes: only SessionMint writes; AuthBroker reads and calls the adapter's existing invalidation hooks; every write carries the policy version read at validation start and the adapter rejects writes whose version is stale (human: ~1.5 days / CC: ~25 min) (recommended)\n ✅ Revocation racing a mint can never resurrect a token; the stale write is rejected and logged\n ✅ Ownership is obvious from the code: one class writes, one class reads, testable with a fake adapter\n ❌ Needs a compare-and-set or version-tag on the adapter write path, a small adapter change\n2B) Single writer only: SessionMint writes, AuthBroker reads and invalidates, no version check (human: ~1 day / CC: ~15 min)\n ✅ Removes the two-writer coupling with no adapter change at all\n ✅ Simpler to explain and diagram than a version scheme\n ❌ Invalidate-then-write-back is still possible inside a single mint that started before the revocation\n2C) Keep both writers, document the race as accepted (human: ~0 / CC: ~0)\n ✅ Zero implementation cost right now\n ✅ Matches the plan as written\n ❌ Ships a known fail-open path in auth; the first incident is a security incident\nNet: You are trading a small adapter change for a guarantee that revocation always wins the race.": "2A Single writer + version (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:25:21.601Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "9e247dab-a089-4c51-ad6d-f067e2d4e5fe",
|
||||
"toolUseId": "toolu_01FAK7TsSk5FC7M297kP2B6L",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D3 — How does the new auth flow reach production?\nProject/branch/task: main — Multi-tenant Auth Refactor; legacyAuthFlow() is rewritten in place (PLAN.md:27-28) with no rollout or fallback named.\nELI10: The plan replaces the login path in one shot. If the new path has a bug, every tenant is locked out at once and the only fix is a revert deploy. A flag that routes some tenants to the new path and the rest to the old one lets you find the bug with one tenant, not all of them.\nStakes if we pick wrong: A global auth outage across all tenants with a deploy-length recovery time instead of a flag flip.\nRecommendation: 3A because auth is the one place where the cost of being wrong is total; make that cost a flag flip. Reversibility preference, strangler fig over big bang.\nCompleteness: 3A=10/10, 3B=7/10, 3C=3/10\nPros / cons:\n3A) Per-tenant flag: legacyAuthFlow() stays callable behind a per-tenant flag, new flow rolls out tenant by tenant, legacy deleted in a follow-up once at 100% (human: ~1 day / CC: ~15 min) (recommended)\n ✅ Rollback is a flag flip per tenant, seconds instead of a deploy\n ✅ Lets you canary on an internal tenant with the regression suite still green on the legacy side\n ❌ Two code paths coexist for one release; legacy removal must actually happen later\n3B) Global kill-switch flag only: new flow on for everyone, one env flag falls back to legacy (human: ~0.5 day / CC: ~10 min)\n ✅ Still a flag flip to recover, no redeploy\n ✅ Less flag plumbing than per-tenant routing\n ❌ First bad tenant takes everyone with it before you flip; no canary\n3C) Big-bang rewrite as planned (human: ~0 / CC: ~0)\n ✅ One code path, nothing to clean up later\n ✅ Smallest diff\n ❌ Recovery is a revert deploy while every tenant is locked out\nNet: You are trading one release of dual code paths for a recovery time measured in seconds instead of deploys.",
|
||||
"header": "Rollout",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "3A Per-tenant flag (recommended)",
|
||||
"description": "Strangler fig: route tenants to new flow incrementally, legacy stays as fallback until 100%."
|
||||
},
|
||||
{
|
||||
"label": "3B Global kill-switch",
|
||||
"description": "New flow on for all; a single flag falls back to legacy."
|
||||
},
|
||||
{
|
||||
"label": "3C Big-bang as planned",
|
||||
"description": "Rewrite legacyAuthFlow() in place, no flag."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D3 — How does the new auth flow reach production?\nProject/branch/task: main — Multi-tenant Auth Refactor; legacyAuthFlow() is rewritten in place (PLAN.md:27-28) with no rollout or fallback named.\nELI10: The plan replaces the login path in one shot. If the new path has a bug, every tenant is locked out at once and the only fix is a revert deploy. A flag that routes some tenants to the new path and the rest to the old one lets you find the bug with one tenant, not all of them.\nStakes if we pick wrong: A global auth outage across all tenants with a deploy-length recovery time instead of a flag flip.\nRecommendation: 3A because auth is the one place where the cost of being wrong is total; make that cost a flag flip. Reversibility preference, strangler fig over big bang.\nCompleteness: 3A=10/10, 3B=7/10, 3C=3/10\nPros / cons:\n3A) Per-tenant flag: legacyAuthFlow() stays callable behind a per-tenant flag, new flow rolls out tenant by tenant, legacy deleted in a follow-up once at 100% (human: ~1 day / CC: ~15 min) (recommended)\n ✅ Rollback is a flag flip per tenant, seconds instead of a deploy\n ✅ Lets you canary on an internal tenant with the regression suite still green on the legacy side\n ❌ Two code paths coexist for one release; legacy removal must actually happen later\n3B) Global kill-switch flag only: new flow on for everyone, one env flag falls back to legacy (human: ~0.5 day / CC: ~10 min)\n ✅ Still a flag flip to recover, no redeploy\n ✅ Less flag plumbing than per-tenant routing\n ❌ First bad tenant takes everyone with it before you flip; no canary\n3C) Big-bang rewrite as planned (human: ~0 / CC: ~0)\n ✅ One code path, nothing to clean up later\n ✅ Smallest diff\n ❌ Recovery is a revert deploy while every tenant is locked out\nNet: You are trading one release of dual code paths for a recovery time measured in seconds instead of deploys.": "3A Per-tenant flag (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:25:42.221Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "9e247dab-a089-4c51-ad6d-f067e2d4e5fe",
|
||||
"toolUseId": "toolu_01Yat2SnnNJje3SELzDVDbAm",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D4 — How should validateAndDispatch() handle errors?\nProject/branch/task: main — Multi-tenant Auth Refactor; validateAndDispatch() is 60 lines, three nested try/catch blocks, each catch swallows a different error class (PLAN.md:23-24).\nELI10: Right now when something goes wrong inside validation, the code catches the problem and keeps going as if nothing happened. In a login path that means a broken check can quietly pass a request through. The fix is to make every failure land in one place that always says no and always writes a log line.\nStakes if we pick wrong: A signature check or tenant lookup that throws becomes an allowed request with no trace in the logs.\nRecommendation: 4A because a flat pipeline with typed errors and one fail-closed boundary is both the safest and the easiest to test branch by branch; explicit over clever.\nCompleteness: 4A=10/10, 4B=7/10, 4C=3/10\nPros / cons:\n4A) Flatten into a linear pipeline of small steps; define typed errors (TokenError, PolicyError, TenantError, IdpError); one catch at the boundary maps every error to an explicit deny result with a structured log and metric; unknown errors also deny (human: ~1 day / CC: ~20 min) (recommended)\n ✅ Every failure path denies and is observable; no branch can silently pass a request\n ✅ Each step is a pure-ish function you can unit test in isolation, and the boundary gets one test per error class\n ❌ Callers that relied on a swallowed error to continue will now see a deny and must be updated\n4B) Keep the structure, stop swallowing: each existing catch rethrows a typed error, one outer catch denies (human: ~0.5 day / CC: ~10 min)\n ✅ Small diff on the existing function, fail-open path is closed\n ✅ No caller behavior change beyond receiving denies where errors were hidden\n ❌ Still 60 lines and three nesting levels; branch coverage stays hard to reason about\n4C) Leave as-is, add logging inside each catch (human: ~0.25 day / CC: ~5 min)\n ✅ Cheapest possible change, purely additive\n ✅ You at least see the swallowed errors in logs\n ❌ Still fail-open; logging a bypass is not preventing one\nNet: You are trading a one-function refactor for an auth dispatcher that cannot pass a request it failed to validate.",
|
||||
"header": "Error handling",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "4A Flatten + typed errors (recommended)",
|
||||
"description": "Linear pipeline, typed error classes, one fail-closed boundary with structured logging."
|
||||
},
|
||||
{
|
||||
"label": "4B Rethrow, one outer catch",
|
||||
"description": "Keep nesting but rethrow typed errors; outer catch denies."
|
||||
},
|
||||
{
|
||||
"label": "4C Log inside each catch",
|
||||
"description": "Leave structure, add logging to the swallowing catches."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D4 — How should validateAndDispatch() handle errors?\nProject/branch/task: main — Multi-tenant Auth Refactor; validateAndDispatch() is 60 lines, three nested try/catch blocks, each catch swallows a different error class (PLAN.md:23-24).\nELI10: Right now when something goes wrong inside validation, the code catches the problem and keeps going as if nothing happened. In a login path that means a broken check can quietly pass a request through. The fix is to make every failure land in one place that always says no and always writes a log line.\nStakes if we pick wrong: A signature check or tenant lookup that throws becomes an allowed request with no trace in the logs.\nRecommendation: 4A because a flat pipeline with typed errors and one fail-closed boundary is both the safest and the easiest to test branch by branch; explicit over clever.\nCompleteness: 4A=10/10, 4B=7/10, 4C=3/10\nPros / cons:\n4A) Flatten into a linear pipeline of small steps; define typed errors (TokenError, PolicyError, TenantError, IdpError); one catch at the boundary maps every error to an explicit deny result with a structured log and metric; unknown errors also deny (human: ~1 day / CC: ~20 min) (recommended)\n ✅ Every failure path denies and is observable; no branch can silently pass a request\n ✅ Each step is a pure-ish function you can unit test in isolation, and the boundary gets one test per error class\n ❌ Callers that relied on a swallowed error to continue will now see a deny and must be updated\n4B) Keep the structure, stop swallowing: each existing catch rethrows a typed error, one outer catch denies (human: ~0.5 day / CC: ~10 min)\n ✅ Small diff on the existing function, fail-open path is closed\n ✅ No caller behavior change beyond receiving denies where errors were hidden\n ❌ Still 60 lines and three nesting levels; branch coverage stays hard to reason about\n4C) Leave as-is, add logging inside each catch (human: ~0.25 day / CC: ~5 min)\n ✅ Cheapest possible change, purely additive\n ✅ You at least see the swallowed errors in logs\n ❌ Still fail-open; logging a bypass is not preventing one\nNet: You are trading a one-function refactor for an auth dispatcher that cannot pass a request it failed to validate.": "4A Flatten + typed errors (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:26:08.879Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "9e247dab-a089-4c51-ad6d-f067e2d4e5fe",
|
||||
"toolUseId": "toolu_01KpcnLtF32eFBUhkb7LecvG",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D5 — One cache-key builder or one per service?\nProject/branch/task: main — Multi-tenant Auth Refactor; after D2, AuthBroker reads and SessionMint writes entries keyed by tenant ID, issuer, audience, and policy version (PLAN.md:7-8).\nELI10: Both services have to spell out the exact same cache key, in the same order, with the same policy version. If they each write their own version of that logic, one typo means the reader looks in a different drawer than the writer filled. Worst case, a key that drops the tenant part lets one tenant read another tenant's cached token.\nStakes if we pick wrong: Silent cache misses at best; a cross-tenant cache hit at worst, with no test that would notice the two builders drifted.\nRecommendation: 5A because the key is a tenant-isolation boundary and DRY matters most exactly there; one builder, one test file, one place to audit.\nCompleteness: 5A=10/10, 5B=7/10, 5C=3/10\nPros / cons:\n5A) Single cacheKeyFor(tenantId, issuer, audience, policyVersion) helper plus a readPolicyVersion() helper, exported from one module, used by both services, with a test asserting every field is present and ordered (human: ~2h / CC: ~5 min) (recommended)\n ✅ Reader and writer cannot drift; the tenant field is structurally required by the signature\n ✅ One property-style test proves two different tenants never produce the same key\n ❌ One more small shared module in the diff\n5B) Shared helper for the key only; each service reads policy version itself (human: ~1h / CC: ~3 min)\n ✅ Key drift eliminated with the smallest shared surface\n ✅ No change to how services obtain policy version\n ❌ Version read can drift, which is the exact input the D2 stale-write check depends on\n5C) Each service builds its own key (human: ~0 / CC: ~0)\n ✅ No shared module, services stay fully independent\n ✅ Nothing to coordinate between the two implementers\n ❌ Two copies of a tenant-isolation boundary, the most expensive DRY violation available\nNet: You are trading a tiny shared module for a guarantee the reader and writer agree on what a tenant is.",
|
||||
"header": "DRY key",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "5A One key + version helper (recommended)",
|
||||
"description": "Shared cacheKeyFor() and readPolicyVersion(), used by both services, tested once."
|
||||
},
|
||||
{
|
||||
"label": "5B Key helper only",
|
||||
"description": "Shared key builder; each service reads policy version itself."
|
||||
},
|
||||
{
|
||||
"label": "5C Per-service keys",
|
||||
"description": "Each service builds its own key and version read."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D5 — One cache-key builder or one per service?\nProject/branch/task: main — Multi-tenant Auth Refactor; after D2, AuthBroker reads and SessionMint writes entries keyed by tenant ID, issuer, audience, and policy version (PLAN.md:7-8).\nELI10: Both services have to spell out the exact same cache key, in the same order, with the same policy version. If they each write their own version of that logic, one typo means the reader looks in a different drawer than the writer filled. Worst case, a key that drops the tenant part lets one tenant read another tenant's cached token.\nStakes if we pick wrong: Silent cache misses at best; a cross-tenant cache hit at worst, with no test that would notice the two builders drifted.\nRecommendation: 5A because the key is a tenant-isolation boundary and DRY matters most exactly there; one builder, one test file, one place to audit.\nCompleteness: 5A=10/10, 5B=7/10, 5C=3/10\nPros / cons:\n5A) Single cacheKeyFor(tenantId, issuer, audience, policyVersion) helper plus a readPolicyVersion() helper, exported from one module, used by both services, with a test asserting every field is present and ordered (human: ~2h / CC: ~5 min) (recommended)\n ✅ Reader and writer cannot drift; the tenant field is structurally required by the signature\n ✅ One property-style test proves two different tenants never produce the same key\n ❌ One more small shared module in the diff\n5B) Shared helper for the key only; each service reads policy version itself (human: ~1h / CC: ~3 min)\n ✅ Key drift eliminated with the smallest shared surface\n ✅ No change to how services obtain policy version\n ❌ Version read can drift, which is the exact input the D2 stale-write check depends on\n5C) Each service builds its own key (human: ~0 / CC: ~0)\n ✅ No shared module, services stay fully independent\n ✅ Nothing to coordinate between the two implementers\n ❌ Two copies of a tenant-isolation boundary, the most expensive DRY violation available\nNet: You are trading a tiny shared module for a guarantee the reader and writer agree on what a tenant is.": "5A One key + version helper (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:26:29.009Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "9e247dab-a089-4c51-ad6d-f067e2d4e5fe",
|
||||
"toolUseId": "toolu_01CvcP53ZXZV83Hx2XKffiaV",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D6 — How deep does test coverage go for the auth flows?\nProject/branch/task: main — Multi-tenant Auth Refactor; plan promises unit and integration coverage for new components only (PLAN.md:14-16); coverage diagram shows 27 gaps, 7 of them flow-level.\nELI10: Unit tests prove each piece works alone. They cannot prove that revoking a token actually stops the next request, or that tenant A can never read tenant B's session, because those bugs live in how the pieces connect. Auth is where connection bugs become breaches.\nStakes if we pick wrong: A green unit suite and a cross-tenant leak or a revoked token that still works in production.\nRecommendation: 6A because with CC the extra flow tests cost minutes, and the paths they cover are exactly the ones a breach would use. Well-tested is non-negotiable.\nCompleteness: 6A=10/10, 6B=7/10, 6C=4/10\nPros / cons:\n6A) Everything in the diagram: unit tests for every branch, the CRITICAL regression suite, plus integration tests for the 7 [→E2E] flows (revocation racing mint, tenant isolation, flag routing, IDP partial failure) against a fake adapter and stubbed IDP (human: ~3 days / CC: ~45 min) (recommended)\n ✅ Every fail-closed path and every tenant boundary is asserted, not assumed\n ✅ The race from D2 gets a deterministic test using an adapter fake that delays the write\n ❌ Largest test diff; needs a fake adapter and an IDP stub harness if none exists yet\n6B) Unit tests for every branch plus the regression suite; skip the integration flows (human: ~1.5 days / CC: ~25 min)\n ✅ Every function branch covered, regression protection in place\n ✅ No new integration harness to build or maintain\n ❌ Revocation race, tenant isolation, and flag routing are only covered by inspection\n6C) Plan as written: success and error paths for new components only, plus the mandatory regression suite (human: ~1 day / CC: ~15 min)\n ✅ Smallest test effort beyond the required regression tests\n ✅ Matches the plan author's stated intent\n ❌ Fail-closed boundary, stale-write rejection, and key collision go untested\nNet: You are trading about 20 extra CC minutes for tests on the exact paths an attacker or an outage would exercise.",
|
||||
"header": "Test depth",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "6A Full diagram incl. E2E (recommended)",
|
||||
"description": "Unit for every branch + regression suite + 7 integration flows with fake adapter and IDP stub."
|
||||
},
|
||||
{
|
||||
"label": "6B Unit + regression only",
|
||||
"description": "Every branch unit-tested plus regression suite; no integration flows."
|
||||
},
|
||||
{
|
||||
"label": "6C Plan as written + regression",
|
||||
"description": "New-component success/error paths plus the mandatory regression suite."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D6 — How deep does test coverage go for the auth flows?\nProject/branch/task: main — Multi-tenant Auth Refactor; plan promises unit and integration coverage for new components only (PLAN.md:14-16); coverage diagram shows 27 gaps, 7 of them flow-level.\nELI10: Unit tests prove each piece works alone. They cannot prove that revoking a token actually stops the next request, or that tenant A can never read tenant B's session, because those bugs live in how the pieces connect. Auth is where connection bugs become breaches.\nStakes if we pick wrong: A green unit suite and a cross-tenant leak or a revoked token that still works in production.\nRecommendation: 6A because with CC the extra flow tests cost minutes, and the paths they cover are exactly the ones a breach would use. Well-tested is non-negotiable.\nCompleteness: 6A=10/10, 6B=7/10, 6C=4/10\nPros / cons:\n6A) Everything in the diagram: unit tests for every branch, the CRITICAL regression suite, plus integration tests for the 7 [→E2E] flows (revocation racing mint, tenant isolation, flag routing, IDP partial failure) against a fake adapter and stubbed IDP (human: ~3 days / CC: ~45 min) (recommended)\n ✅ Every fail-closed path and every tenant boundary is asserted, not assumed\n ✅ The race from D2 gets a deterministic test using an adapter fake that delays the write\n ❌ Largest test diff; needs a fake adapter and an IDP stub harness if none exists yet\n6B) Unit tests for every branch plus the regression suite; skip the integration flows (human: ~1.5 days / CC: ~25 min)\n ✅ Every function branch covered, regression protection in place\n ✅ No new integration harness to build or maintain\n ❌ Revocation race, tenant isolation, and flag routing are only covered by inspection\n6C) Plan as written: success and error paths for new components only, plus the mandatory regression suite (human: ~1 day / CC: ~15 min)\n ✅ Smallest test effort beyond the required regression tests\n ✅ Matches the plan author's stated intent\n ❌ Fail-closed boundary, stale-write rejection, and key collision go untested\nNet: You are trading about 20 extra CC minutes for tests on the exact paths an attacker or an outage would exercise.": "6A Full diagram incl. E2E (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:27:09.786Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "9e247dab-a089-4c51-ad6d-f067e2d4e5fe",
|
||||
"toolUseId": "toolu_01KAQW9LBcZnJQcqUNdCwVXA",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D7 — Parallelize the 5 IDP calls, and how?\nProject/branch/task: main — Multi-tenant Auth Refactor; token validation issues 5 sequential IDP calls the plan says are independent (PLAN.md:31-32).\nELI10: Every login waits for five round trips to the identity provider one after another. Firing them at once cuts login latency to roughly one round trip. But firing five at once per login also multiplies the load you put on the identity provider, and two of those five (discovery document, signing keys) rarely change and can be cached instead of fetched at all.\nStakes if we pick wrong: Either logins stay 5x slower than needed, or you trip the identity provider's rate limit under load and every tenant sees intermittent login failures.\nRecommendation: 7A because the fastest IDP call is the one you do not make; parallelize what remains, bound it with a timeout, and fail closed on any rejection.\nCompleteness: 7A=10/10, 7B=7/10, 7C=3/10\nPros / cons:\n7A) Cache the static IDP responses (discovery, JWKS) with TTL in the existing adapter; run the remaining calls with Promise.all wrapped in a per-call timeout; any rejection denies with all outcomes logged (human: ~1 day / CC: ~20 min) (recommended)\n ✅ Login latency drops to one round trip and IDP request volume drops by up to 40 percent\n ✅ A hung IDP call cannot hang the login; the timeout converts it into a clean deny\n ❌ Needs a TTL policy and a test for stale-key rotation on the cached JWKS\n7B) Promise.all on all 5 calls, no caching, no timeout wrapper (human: ~1h / CC: ~3 min)\n ✅ Trivial change, immediate latency win as the plan describes\n ✅ Fail-fast on first rejection matches the required-all semantics of auth\n ❌ 5x concurrent IDP load per login and a slow call still stalls the whole request\n7C) Leave sequential (human: ~0 / CC: ~0)\n ✅ Zero risk of new concurrency behavior\n ✅ IDP load pattern unchanged\n ❌ Every login pays five serial round trips for no reason\nNet: You are trading a TTL cache and a timeout wrapper for logins that are both faster and safer under IDP degradation.",
|
||||
"header": "Performance",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "7A Cache static + Promise.all + timeout (recommended)",
|
||||
"description": "TTL-cache discovery/JWKS; parallelize the rest with per-call timeout; any rejection denies."
|
||||
},
|
||||
{
|
||||
"label": "7B Promise.all only",
|
||||
"description": "Parallelize all 5 calls as the plan states, no caching or timeout."
|
||||
},
|
||||
{
|
||||
"label": "7C Leave sequential",
|
||||
"description": "No change to the IDP call pattern."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D7 — Parallelize the 5 IDP calls, and how?\nProject/branch/task: main — Multi-tenant Auth Refactor; token validation issues 5 sequential IDP calls the plan says are independent (PLAN.md:31-32).\nELI10: Every login waits for five round trips to the identity provider one after another. Firing them at once cuts login latency to roughly one round trip. But firing five at once per login also multiplies the load you put on the identity provider, and two of those five (discovery document, signing keys) rarely change and can be cached instead of fetched at all.\nStakes if we pick wrong: Either logins stay 5x slower than needed, or you trip the identity provider's rate limit under load and every tenant sees intermittent login failures.\nRecommendation: 7A because the fastest IDP call is the one you do not make; parallelize what remains, bound it with a timeout, and fail closed on any rejection.\nCompleteness: 7A=10/10, 7B=7/10, 7C=3/10\nPros / cons:\n7A) Cache the static IDP responses (discovery, JWKS) with TTL in the existing adapter; run the remaining calls with Promise.all wrapped in a per-call timeout; any rejection denies with all outcomes logged (human: ~1 day / CC: ~20 min) (recommended)\n ✅ Login latency drops to one round trip and IDP request volume drops by up to 40 percent\n ✅ A hung IDP call cannot hang the login; the timeout converts it into a clean deny\n ❌ Needs a TTL policy and a test for stale-key rotation on the cached JWKS\n7B) Promise.all on all 5 calls, no caching, no timeout wrapper (human: ~1h / CC: ~3 min)\n ✅ Trivial change, immediate latency win as the plan describes\n ✅ Fail-fast on first rejection matches the required-all semantics of auth\n ❌ 5x concurrent IDP load per login and a slow call still stalls the whole request\n7C) Leave sequential (human: ~0 / CC: ~0)\n ✅ Zero risk of new concurrency behavior\n ✅ IDP load pattern unchanged\n ❌ Every login pays five serial round trips for no reason\nNet: You are trading a TTL cache and a timeout wrapper for logins that are both faster and safer under IDP degradation.": "7A Cache static + Promise.all + timeout (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:27:50.057Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "9e247dab-a089-4c51-ad6d-f067e2d4e5fe",
|
||||
"toolUseId": "toolu_013HN2XTYK2fZtsbSTUWzpEc",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D8 — TODO: remove legacyAuthFlow() and the per-tenant flag after 100 percent rollout?\nProject/branch/task: main — Multi-tenant Auth Refactor; D3 keeps legacyAuthFlow() alive behind a per-tenant flag for the rollout release.\nELI10: Once every tenant runs on the new flow, the old login code and its flag are dead weight that someone will eventually be afraid to delete. Writing the removal down now, with the exit condition, keeps the strangler fig from becoming a permanent second code path.\nStakes if we pick wrong: Two auth paths live forever, and every future auth change has to be made and tested twice.\nRecommendation: 8A because the deletion is not this PR's job but it is this PR's debt; capture it with its trigger so it actually happens.\nNote: options differ in kind, not coverage — no completeness score.\nWhat: Delete legacyAuthFlow(), the per-tenant new-flow flag, and the legacy branch of the regression suite once all tenants are on the new flow.\nWhy: Two auth code paths double the test and review cost of every future auth change.\nPros: One auth path; the regression suite collapses to the new flow only.\nCons: Must wait for 100 percent rollout plus a bake period; deleting early removes the rollback.\nContext: D3 chose a strangler-fig rollout. The regression suite from Section 3 runs against both paths during rollout. Exit condition: all tenants flagged on, no flag flips for one full release cycle.\nDepends on: 100 percent tenant rollout of the new flow.\nPros / cons:\n8A) Add to TODOS.md with the exit condition above (human: ~5 min / CC: ~1 min) (recommended)\n ✅ The debt and its trigger are written down where /retro and /ship will surface it\n ✅ Nobody has to remember why the flag exists six months from now\n ❌ TODOS.md does not exist yet in this repo, so this creates it\n8B) Skip, not valuable enough (human: ~0 / CC: ~0)\n ✅ No new file in the repo\n ✅ Team may already track cleanup elsewhere\n ❌ Legacy path and flag risk becoming permanent by default\n8C) Build it now: delete legacyAuthFlow() in this PR (human: ~0 / CC: ~0)\n ✅ One code path immediately, smallest long-term surface\n ✅ No follow-up to track\n ❌ Reverses D3; removes the rollback path before the new flow has seen production\nNet: You are trading one TODO entry for a guaranteed end date on the dual-path release.",
|
||||
"header": "TODO",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "8A Add to TODOS.md (recommended)",
|
||||
"description": "Capture legacy removal with its 100 percent rollout exit condition."
|
||||
},
|
||||
{
|
||||
"label": "8B Skip",
|
||||
"description": "Do not record the follow-up."
|
||||
},
|
||||
{
|
||||
"label": "8C Build it now",
|
||||
"description": "Delete legacyAuthFlow() in this PR, reversing D3."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D8 — TODO: remove legacyAuthFlow() and the per-tenant flag after 100 percent rollout?\nProject/branch/task: main — Multi-tenant Auth Refactor; D3 keeps legacyAuthFlow() alive behind a per-tenant flag for the rollout release.\nELI10: Once every tenant runs on the new flow, the old login code and its flag are dead weight that someone will eventually be afraid to delete. Writing the removal down now, with the exit condition, keeps the strangler fig from becoming a permanent second code path.\nStakes if we pick wrong: Two auth paths live forever, and every future auth change has to be made and tested twice.\nRecommendation: 8A because the deletion is not this PR's job but it is this PR's debt; capture it with its trigger so it actually happens.\nNote: options differ in kind, not coverage — no completeness score.\nWhat: Delete legacyAuthFlow(), the per-tenant new-flow flag, and the legacy branch of the regression suite once all tenants are on the new flow.\nWhy: Two auth code paths double the test and review cost of every future auth change.\nPros: One auth path; the regression suite collapses to the new flow only.\nCons: Must wait for 100 percent rollout plus a bake period; deleting early removes the rollback.\nContext: D3 chose a strangler-fig rollout. The regression suite from Section 3 runs against both paths during rollout. Exit condition: all tenants flagged on, no flag flips for one full release cycle.\nDepends on: 100 percent tenant rollout of the new flow.\nPros / cons:\n8A) Add to TODOS.md with the exit condition above (human: ~5 min / CC: ~1 min) (recommended)\n ✅ The debt and its trigger are written down where /retro and /ship will surface it\n ✅ Nobody has to remember why the flag exists six months from now\n ❌ TODOS.md does not exist yet in this repo, so this creates it\n8B) Skip, not valuable enough (human: ~0 / CC: ~0)\n ✅ No new file in the repo\n ✅ Team may already track cleanup elsewhere\n ❌ Legacy path and flag risk becoming permanent by default\n8C) Build it now: delete legacyAuthFlow() in this PR (human: ~0 / CC: ~0)\n ✅ One code path immediately, smallest long-term surface\n ✅ No follow-up to track\n ❌ Reverses D3; removes the rollback path before the new flow has seen production\nNet: You are trading one TODO entry for a guaranteed end date on the dual-path release.": "8A Add to TODOS.md (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:28:20.760Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "9e247dab-a089-4c51-ad6d-f067e2d4e5fe",
|
||||
"toolUseId": "toolu_019cfb6mC6vMWudY5ZukLHzh",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D9 — Which auth path runs when the per-tenant flag store is unreachable?\nProject/branch/task: main — Multi-tenant Auth Refactor; D3 routes each tenant to the new flow or legacyAuthFlow() by a per-tenant flag.\nELI10: The flag lookup is one more thing that can fail during login. If the flag service times out, the code has to pick a path anyway. Picking the old, proven path keeps users logged in during a flag outage; denying everyone is safer against a half-configured new flow but turns a flag outage into an auth outage.\nStakes if we pick wrong: Either a flag-service blip logs every tenant out, or a tenant already migrated silently falls back to a legacy path that no longer matches its config.\nRecommendation: 9A because the legacy path is the known-good path for the whole rollout window, and the fallback is loud, bounded, and tested.\nCompleteness: 9A=10/10, 9B=7/10, 9C=3/10\nPros / cons:\n9A) Fall back to legacyAuthFlow() on flag lookup failure, with a short lookup timeout, a warn-level structured log, a metric, and an integration test that stubs the flag store as down (human: ~3h / CC: ~8 min) (recommended)\n ✅ A flag-store outage degrades to yesterday's behavior instead of an auth outage\n ✅ The fallback is visible in metrics so a stuck fallback cannot hide for days\n ❌ Tenants already migrated briefly run legacy during the outage; acceptable only while legacy still exists\n9B) Fall back to legacy, no dedicated test or metric (human: ~1h / CC: ~3 min)\n ✅ Same availability behavior with less plumbing\n ✅ Smallest diff to the router\n ❌ A permanently failing flag lookup would silently pin everyone to legacy\n9C) Deny on flag lookup failure (human: ~1h / CC: ~3 min)\n ✅ Strictly fail-closed; no request ever runs a path it was not configured for\n ✅ Simplest to reason about from a security standpoint\n ❌ Converts a flag-service blip into a total login outage for all tenants\nNet: You are trading a few lines of timeout and metric plumbing for a flag outage that users never notice.",
|
||||
"header": "Flag failure",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "9A Legacy fallback + metric + test (recommended)",
|
||||
"description": "Timeout on lookup, fall back to legacyAuthFlow(), warn log, metric, stubbed-down integration test."
|
||||
},
|
||||
{
|
||||
"label": "9B Legacy fallback only",
|
||||
"description": "Fall back to legacy on lookup failure, no metric or dedicated test."
|
||||
},
|
||||
{
|
||||
"label": "9C Deny on failure",
|
||||
"description": "Fail closed: deny every request while the flag store is unreachable."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D9 — Which auth path runs when the per-tenant flag store is unreachable?\nProject/branch/task: main — Multi-tenant Auth Refactor; D3 routes each tenant to the new flow or legacyAuthFlow() by a per-tenant flag.\nELI10: The flag lookup is one more thing that can fail during login. If the flag service times out, the code has to pick a path anyway. Picking the old, proven path keeps users logged in during a flag outage; denying everyone is safer against a half-configured new flow but turns a flag outage into an auth outage.\nStakes if we pick wrong: Either a flag-service blip logs every tenant out, or a tenant already migrated silently falls back to a legacy path that no longer matches its config.\nRecommendation: 9A because the legacy path is the known-good path for the whole rollout window, and the fallback is loud, bounded, and tested.\nCompleteness: 9A=10/10, 9B=7/10, 9C=3/10\nPros / cons:\n9A) Fall back to legacyAuthFlow() on flag lookup failure, with a short lookup timeout, a warn-level structured log, a metric, and an integration test that stubs the flag store as down (human: ~3h / CC: ~8 min) (recommended)\n ✅ A flag-store outage degrades to yesterday's behavior instead of an auth outage\n ✅ The fallback is visible in metrics so a stuck fallback cannot hide for days\n ❌ Tenants already migrated briefly run legacy during the outage; acceptable only while legacy still exists\n9B) Fall back to legacy, no dedicated test or metric (human: ~1h / CC: ~3 min)\n ✅ Same availability behavior with less plumbing\n ✅ Smallest diff to the router\n ❌ A permanently failing flag lookup would silently pin everyone to legacy\n9C) Deny on flag lookup failure (human: ~1h / CC: ~3 min)\n ✅ Strictly fail-closed; no request ever runs a path it was not configured for\n ✅ Simplest to reason about from a security standpoint\n ❌ Converts a flag-service blip into a total login outage for all tenants\nNet: You are trading a few lines of timeout and metric plumbing for a flag outage that users never notice.": "9A Legacy fallback + metric + test (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-10T18:29:01.552Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -10,6 +10,8 @@ interface ArtifactPermissionContext {
|
||||
cwd: string;
|
||||
/** Set only by the launcher that created HOME/.gstack, never from ambient env. */
|
||||
ownedStateRoot?: string;
|
||||
/** Optional native plans root supplied by the same isolated launcher. */
|
||||
ownedNativePlansRoot?: string;
|
||||
commandStartedAt: number;
|
||||
now?: number;
|
||||
transcriptStatus: string;
|
||||
@@ -213,6 +215,50 @@ export function autoplanArtifactPermissionInput(
|
||||
export const autoplanArtifactMenuKey = (viewport: string) =>
|
||||
`menu:${createHash('sha256').update(viewport.replace(/\r\n?/g, '\n')).digest('hex')}`;
|
||||
|
||||
/** Queued native-plan edits remain unstarted; this never grants their input. */
|
||||
function ownedQueuedNativePlan(file: unknown, root: string | undefined, pendingTime: number): file is string {
|
||||
if (!root || typeof file !== 'string' || !path.isAbsolute(root) || path.resolve(root) !== root ||
|
||||
!path.isAbsolute(file) || path.resolve(file) !== file || path.dirname(file) !== root ||
|
||||
!/^[a-z][a-z0-9-]*\.md$/.test(path.basename(file))) return false;
|
||||
try {
|
||||
return fs.lstatSync(root).isDirectory() && fs.lstatSync(file).isFile() &&
|
||||
fs.realpathSync(file) === path.join(fs.realpathSync(root), path.basename(file)) && fs.statSync(file).size <= MAX_BYTES &&
|
||||
Math.floor(fs.statSync(file).mtimeMs) <= pendingTime;
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
/** Bind completed snapshot output and queued native-plan redraw labels before
|
||||
* the full current panel. Arbitrary prose, examples and competing panels stay. */
|
||||
function queuedPlanViewport(viewport: string, queuedPlans: number, current: NativePublicToolEvent,
|
||||
events: NativePublicToolEvent[]): string {
|
||||
if (!queuedPlans) return viewport;
|
||||
const lines = viewport.replace(/\r\n?/g, '\n').split('\n');
|
||||
const title = lines.findIndex(line => /^[●⏺] Update\(/.test(line));
|
||||
if (title < 0 || lines.filter(line => /^[●⏺] Update\(/.test(line)).length !== 1) return viewport;
|
||||
const panel = lines.findIndex((line, i) => i > title && /^[─╌]{8,}$/.test(line));
|
||||
const redraws = lines.slice(title + 1, panel).filter(line => line.trim());
|
||||
if (panel < 0 || redraws.length !== queuedPlans || redraws.some(line => !/^[●⏺] Updated plan$/.test(line))) return viewport;
|
||||
const prefix = lines.slice(0, title);
|
||||
while (prefix.at(-1)?.trim() === '') prefix.pop();
|
||||
if (prefix.some(line => line.trim())) {
|
||||
const prior = events.slice(0, events.indexOf(current));
|
||||
const result = prior.filter(event => event.kind === 'result').at(-1);
|
||||
const use = result && prior.find(event => event.kind === 'use' && event.toolUseId === result.toolUseId);
|
||||
const command = /^ {8}"([^"\n]+)…\)$/.exec(prefix[0] ?? '');
|
||||
const outputEnd = prefix.length - 2;
|
||||
if (!command || !use || use.name !== 'Bash' || use.messageId !== current.messageId ||
|
||||
use.requestId !== current.requestId || typeof use.input?.command !== 'string' ||
|
||||
!use.input.command.includes(command[1]!) || result?.isError !== false || typeof result.content !== 'string' ||
|
||||
!/^ {2}⎿[ \u00a0]+\{$/.test(prefix[1] ?? '') ||
|
||||
!/^ {5}… \+[1-9]\d* lines \(ctrl\+o to expand\)$/.test(prefix[outputEnd] ?? '') ||
|
||||
!/^ {2}⎿[ \u00a0]+Allowed by auto mode classifier$/.test(prefix.at(-1) ?? '') || outputEnd < 3) return viewport;
|
||||
const displayed = ['{', ...prefix.slice(2, outputEnd).map(line => /^ {5}( {2}\S.*)$/.exec(line)?.[1])];
|
||||
if (displayed.some(line => line === undefined) ||
|
||||
result.content.split('\n').slice(0, displayed.length).join('\n') !== displayed.join('\n')) return viewport;
|
||||
}
|
||||
return [lines[title], '', ...lines.slice(panel)].join('\n');
|
||||
}
|
||||
|
||||
/** Metadata-only fallback. Added rows are display evidence, never request content. */
|
||||
export function pendingAutoplanArtifactPermissionInput(viewport: string,
|
||||
context: ArtifactPermissionContext & { pending?: PendingAutoplanArtifact; viewportCapturedAt: number },
|
||||
@@ -327,18 +373,29 @@ export function publishedAutoplanArtifactPermissionInput(viewport: string,
|
||||
typeof input.new_string!=='string' || input.new_string===input.old_string ||
|
||||
(input.replace_all!==undefined&&input.replace_all!==false)) return null;
|
||||
const queued=new Set<string>();
|
||||
let queuedPlans = 0;
|
||||
for (const mutation of [...uses.values()].filter(e=>e.name==='Edit'||e.name==='Write')) {
|
||||
const result=results.get(mutation.toolUseId);
|
||||
if (Date.parse(mutation.timestamp)>pendingTime || (result&&Date.parse(result.timestamp)>pendingTime)) return null;
|
||||
if (result && (Date.parse(mutation.timestamp)>pendingTime || Date.parse(result.timestamp)>pendingTime)) return null;
|
||||
if (mutation.toolUseId===p.toolUseId || result) continue;
|
||||
// Later publications are queued only when this exact batch owns them and
|
||||
// the recorder has not started them. They never supply current authority.
|
||||
if (mutation.name!=='Edit' || mutation.input?.file_path!==p.file ||
|
||||
const nativePlan = mutation.input?.file_path !== p.file &&
|
||||
ownedQueuedNativePlan(mutation.input?.file_path, context.ownedNativePlansRoot, pendingTime);
|
||||
if (mutation.name!=='Edit' || (mutation.input?.file_path!==p.file && !nativePlan) ||
|
||||
typeof mutation.input.old_string!=='string' || !mutation.input.old_string ||
|
||||
typeof mutation.input.new_string!=='string' || mutation.input.old_string===mutation.input.new_string ||
|
||||
(mutation.input.replace_all!==undefined && mutation.input.replace_all!==false) ||
|
||||
mutation.messageId!==current.messageId || mutation.requestId!==current.requestId ||
|
||||
Date.parse(mutation.timestamp)>context.viewportCapturedAt ||
|
||||
events.indexOf(mutation)<=events.indexOf(current) || p.hookSeenIds.includes(mutation.toolUseId)) return null;
|
||||
if (nativePlan) {
|
||||
if (![...uses.values()].some(previous => (previous.name === 'Write' || previous.name === 'Edit') &&
|
||||
previous.input?.file_path === mutation.input!.file_path &&
|
||||
results.get(previous.toolUseId)?.isError === false &&
|
||||
Date.parse(results.get(previous.toolUseId)!.timestamp) <= pendingTime)) return null;
|
||||
queuedPlans++;
|
||||
}
|
||||
queued.add(mutation.toolUseId);
|
||||
}
|
||||
try {
|
||||
@@ -347,7 +404,7 @@ export function publishedAutoplanArtifactPermissionInput(viewport: string,
|
||||
if (!actual || actual.beforeSHA256!==expected.beforeSHA256 || actual.requestSHA256!==expected.requestSHA256 ||
|
||||
JSON.stringify(actual.oldLineHashes)!==JSON.stringify(expected.oldLineHashes) ||
|
||||
JSON.stringify(actual.newLineHashes)!==JSON.stringify(expected.newLineHashes)) return null;
|
||||
return autoplanArtifactPermissionInput(viewport,{...context,
|
||||
return autoplanArtifactPermissionInput(queuedPlanViewport(viewport,queuedPlans,current,events),{...context,
|
||||
publicTools:events.filter(e=>!queued.has(e.toolUseId))},seen);
|
||||
} catch { return null; }
|
||||
}
|
||||
@@ -41,6 +41,7 @@ import { createAutoplanArtifactRecorder } from './autoplan-artifact-recorder';
|
||||
import { trustDialogInput } from './pty-trust-dialog';
|
||||
import { createPtyScreen } from './pty-screen';
|
||||
import { isRecordedDxManualNavigation } from './dx-selected-navigation';
|
||||
import { engCacheWriterDecision } from './eng-cache-writer-decision';
|
||||
|
||||
/** Strip ANSI escapes for pattern-matching against visible text. */
|
||||
export function stripAnsi(s: string): string {
|
||||
@@ -3080,6 +3081,56 @@ function engNumberedFindingAUQ(fp: AskUserQuestionFingerprint): boolean {
|
||||
fp.options.length !== q.options.length || !fp.options.every((o, i) => o.index === i + 1 && o.label === q.options[i]!.label) ||
|
||||
/<gstack-qid/i.test(q.question)) return false;
|
||||
const title = q.question.split('\n')[0]!.replace(/^D[1-9]\d*\s*[—–:-]\s*/i, '');
|
||||
// A declarative severity-labelled issue can own the same cache repair as
|
||||
// an interrogative title. Require its native decision, current assessment,
|
||||
// and opposed implementation options; metadata alone never opens review.
|
||||
const declaredCache = /^(?:Issue|Finding) ([1-9]\d*) \[P[0-3]\] \(confidence (?:10|[1-9])\/10\) [A-Za-z][\w./-]*:[1-9]\d*(?:-[1-9]\d*)?: ([A-Za-z_$][\w$]*) and ([A-Za-z_$][\w$]*) share a global mutable ([A-Za-z_$][\w$]*) via module-level export, and both mutate it\.$/.exec(title);
|
||||
if (declaredCache) {
|
||||
if (!/^Shared cache$/i.test(q.header.trim()) || declaredCache[2] === declaredCache[3]) return false;
|
||||
const escaped = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const escapedCache = escaped(declaredCache[4]!);
|
||||
const decision = /^D([1-9]\d*)\s*[—–:-]/.exec(q.question)?.[1];
|
||||
const owner = `(?:(?:this|the|that) (?:finding|issue|gap|remedy|amendment|assessment|option|risk)|(?:Issue|Finding) ${declaredCache[1]}${decision ? `|D${decision}` : ''})`;
|
||||
const boundary = '(?:^|[.!?;]\\s+|\\n|[✅❌]\\s*)(?:Correction:\\s*)?';
|
||||
const scalarOwner = new RegExp(`${boundary}${owner} (?:is|was|has been) (?:(?:now|already) )?$`, 'i');
|
||||
const current = (value: string) => value
|
||||
.replace(/```[\s\S]*?(?:```|$)|~~~[\s\S]*?(?:~~~|$)/g, '')
|
||||
.replace(/^(?:\s*>| {4}|\t).*$/gm, '')
|
||||
.replace(/"[^"\n]*"|“[^”\n]*”|(?<![A-Za-z0-9])'[^'\n]*'(?![A-Za-z0-9])|‘[^’\n]*’|`(?:withdrawn|superseded|rejected|cancelled|canceled|resolved|closed|not current|no longer current)`/gi,
|
||||
(quoted: string, index: number, source: string) =>
|
||||
/^(?:withdrawn|superseded|rejected|cancelled|canceled|resolved|closed|not current|no longer current)$/i.test(quoted.slice(1, -1)) &&
|
||||
scalarOwner.test(source.slice(0, index)) ? quoted.slice(1, -1) : '')
|
||||
.replace(/`([^`\n]*)`/g, (_, code: string) => /^[A-Za-z_$][\w$]*$/.test(code) ? code : '')
|
||||
.replace(/\*\*/g, '');
|
||||
const framed = /\b(?:source|quoted|historical|hypothetical|earlier|previous)\s+(?:review\s+)?(?:example|excerpt|assessment|finding|material|text)\b|(?:^|[.!?;:]\s+|\n|[✅❌]\s*)(?:if|when|unless|provided|assuming|suppose|imagine|source|example)\b/i;
|
||||
const closed = new RegExp(`${boundary}${owner} (?:is|was|has been) (?:(?:now|already) )?(?:withdrawn|superseded|rejected|cancelled|canceled|resolved|closed|hypothetical|not current|no longer current)\\b|${boundary}no current (?:gap|risk|finding) (?:remains|exists)\\b`, 'i');
|
||||
const removed = new RegExp(`${boundary}(?:(?:${escapedCache}|(?:the|this|that) (?:cache|export|global|writers?|writes?|services?)) (?:is|are|has been|have been) (?:no longer (?:global|mutable|shared|unordered)|(?:now |already )?(?:ordered|serialized|removed))|(?:${escaped(declaredCache[2]!)}|${escaped(declaredCache[3]!)}|(?:the|this|both) services?) no longer (?:share|mutate|write)[a-z]*)\\b`, 'i');
|
||||
const cancelled = /(?:^|[.!?;]\s+|\n|\bCorrection:\s*)(?:do not|don't|never|skip|cancel|withdraw) (?:inject|receive|remove|delete|serialize|order|keep|proceed|accept)\b/i;
|
||||
const inactive = (value: string) => framed.test(value) || closed.test(value) || cancelled.test(value);
|
||||
const text = current(q.question), assessment = [...text.matchAll(/^ELI10: (.+)$/gm)];
|
||||
const preface = text.slice(text.indexOf('\n') + 1, assessment[0]?.index ?? 0).trim().split('\n').filter(Boolean);
|
||||
if (assessment.length !== 1 || preface.length !== 1 ||
|
||||
!/^Project\/branch\/task: \S[^\n]*\bSection [1-9]\d* Architecture\b/.test(preface[0]!) ||
|
||||
!/^Two services write to the same cache through a global variable\./.test(assessment[0]![1]!) ||
|
||||
!/\b(?:writes can interleave|no ordering|unordered writes)\b/.test(assessment[0]![1]!) || inactive(text) ||
|
||||
removed.test(text)) return false;
|
||||
const optionIds = q.options.map(o => /^([1-9]\d*)([A-D])[:.)]\s+(\S[\s\S]*)$/.exec(o.label));
|
||||
if (optionIds.some(id => id?.[1] !== declaredCache[1]) || new Set(optionIds.map(id => id![2])).size !== q.options.length) return false;
|
||||
const actions = optionIds.map(id => id![3]!.replace(/\s*\(recommended\)$/i, ''));
|
||||
return q.options.some((remedy, index) => {
|
||||
if (!new RegExp(`^Inject ${escapedCache};`).test(actions[index]!)) return false;
|
||||
const body = current(remedy.description ?? '').trim();
|
||||
if (inactive(body) ||
|
||||
!new RegExp(`^✅\\s*Both services receive the one ${escapedCache} instance via constructor; the module export goes away\\b`).test(body) ||
|
||||
!new RegExp(`✅\\s*${escapedCache} exposes only [^✅❌.!?]{1,180} and serializes writes per key\\b`).test(body) ||
|
||||
/\b(?:module export (?:stays|remains|is retained)|writes (?:remain|stay) unordered|(?:do not|never) serialize)\b/i.test(body)) return false;
|
||||
return q.options.some((opposed, other) => other !== index && /^Proceed as written$/i.test(actions[other]!) &&
|
||||
!inactive(current(opposed.description ?? '')) && !removed.test(current(opposed.description ?? '')) &&
|
||||
new RegExp(`❌\\s*Ships an? (?:auth cache|${escapedCache}) with two unordered writers and shared test state\\b`).test(current(opposed.description ?? '')) &&
|
||||
!/\b(?:risk|race) (?:is |has been )?(?:resolved|closed|fixed)|\bno (?:race|risk) remains\b/i.test(current(opposed.description ?? '')));
|
||||
});
|
||||
}
|
||||
if (engCacheWriterDecision(q)) return true;
|
||||
// The category may precede the issue number. Bind this library choice to
|
||||
// the current scheduling defect and both concrete outcomes, not its label.
|
||||
const libraryHooks = /^Architecture issue ([1-9]\d*): custom inline scheduler vs the job library's built-in retry hooks\?$/i.exec(title);
|
||||
|
||||
@@ -182,7 +182,7 @@ function diagramBlocks(output: string): string[][] {
|
||||
}
|
||||
|
||||
function treeRow(line: string): { depth: number; text: string } | undefined {
|
||||
const match = /^([ |│]*)(?:[├└]─+|[+|]-+)\s+(.+)$/.exec(line);
|
||||
const match = /^([ |│]*)(?:[├└]─+►?|[+|]-+)\s+(.+)$/.exec(line);
|
||||
if (!match) {
|
||||
// Unindented function roots own following branch rows. Other function
|
||||
// roots also end a subtree, so a sibling cannot lend a coverage marker.
|
||||
@@ -197,7 +197,11 @@ function diagramLegend(lines: string[], firstRow: number): Map<string, boolean>
|
||||
const meanings = new Map<string, boolean>();
|
||||
const pair = String.raw`\[([✓✔✗✘])\][\t ]+(TESTED|COVERED|GAP|UNTESTED)`;
|
||||
const legend = new RegExp(String.raw`^${pair}[\t |,;]+${pair}$`, 'i');
|
||||
for (const line of lines.slice(0, firstRow)) {
|
||||
for (const original of lines.slice(0, firstRow)) {
|
||||
// Decorative branch keys and an explicit GAP explanation do not change
|
||||
// the two coverage meanings. All other qualifiers keep the closed grammar.
|
||||
const line = original.replace(/[\t ]+[─-]+►[\t ]+branch$/i, '')
|
||||
.replace(/(\[[✓✔✗✘]\][\t ]+(?:GAP|UNTESTED))[\t ]+\(no test\)$/i, '$1');
|
||||
const start = line.search(/\[[✓✔✗✘]\]/);
|
||||
if (start < 0) continue;
|
||||
if (/^\s*>|["“”]|\b(?:not|no|never|example|sample|false|incorrect|hypothetical)\b/i.test(line)) return new Map();
|
||||
@@ -222,9 +226,14 @@ function seededDiagram(output: string): boolean {
|
||||
for (const lines of diagramBlocks(output)) {
|
||||
const rows = lines.map(treeRow);
|
||||
const legend = diagramLegend(lines, rows.findIndex(row => row !== undefined));
|
||||
const symbolMeans = (line: string, covered: boolean) =>
|
||||
!/\b(?:not|never)\s+\[[✓✔✗✘]\]|(?:\[[✓✔✗✘]\]|\b(?:marker|symbol))\s+(?:is|are)\s+(?:false|incorrect|wrong)\b/i.test(line) &&
|
||||
[...line.matchAll(/\[([✓✔✗✘])\]/g)].some(match => legend.get(match[1]!) === covered);
|
||||
const symbolMeans = (line: string, covered: boolean) => {
|
||||
if (/\b(?:not|never)\s+\[[✓✔✗✘]\]|(?:\[[✓✔✗✘]\]|\b(?:marker|symbol))\s+(?:is|are)\s+(?:false|incorrect|wrong)\b/i.test(line)) return false;
|
||||
// A status correction [covered]→[gap] carries only its final marker.
|
||||
// Unrelated contradictory markers cannot supply both coverage states.
|
||||
const corrected = line.replace(/\[[✓✔✗✘]\][\t ]*(?:→|->)[\t ]*(?=\[[✓✔✗✘]\])/g, '');
|
||||
const states = [...corrected.matchAll(/\[([✓✔✗✘])\]/g)].map(match => legend.get(match[1]!));
|
||||
return states.includes(covered) && !states.includes(!covered);
|
||||
};
|
||||
const payment = rows.findIndex(row => row && /^processPayment\b/.test(row.text));
|
||||
const refund = rows.findIndex(row => row && /^refundPayment\b/.test(row.text));
|
||||
if (payment < 0 || refund < 0) continue;
|
||||
@@ -238,11 +247,11 @@ function seededDiagram(output: string): boolean {
|
||||
return texts;
|
||||
};
|
||||
const covered = subtree(payment).some(line =>
|
||||
(/(?:\bTESTED\b|\bCOVERED\b)/i.test(line) || symbolMeans(line, true) ||
|
||||
(/✓/.test(line) && !/\[✓\]/.test(line) && (legend.get('✓') ?? true))) && /happy|success|valid|USD/i.test(line) &&
|
||||
(/\[[✓✔✗✘]\]/.test(line) ? symbolMeans(line, true) :
|
||||
/(?:\bTESTED\b|\bCOVERED\b)/i.test(line) || (/✓/.test(line) && (legend.get('✓') ?? true))) && /happy|success|valid|USD/i.test(line) &&
|
||||
!/untested|(?:not|never)\s+(?:yet\s+)?(?:tested|covered)|no\s+test/i.test(line));
|
||||
const missing = subtree(refund).some(line =>
|
||||
(/(?:\[GAP\]|✗\s*GAP|\bUNTESTED\b)/i.test(line) || symbolMeans(line, false)) &&
|
||||
(/\[[✓✔✗✘]\]/.test(line) ? symbolMeans(line, false) : /(?:\[GAP\]|✗\s*GAP|\bUNTESTED\b)/i.test(line)) &&
|
||||
!/\b(?:not|never)\s+(?:\[)?(?:untested|gap)\b|\b(?:untested|gap)\]?\s+(?:is|are)\s+(?:false|incorrect|wrong)\b|\bno\s+(?:coverage\s+)?gaps?\b|\b(?:fully|completely)\s+(?:tested|covered)\b/i.test(line));
|
||||
if (covered && missing) return true;
|
||||
}
|
||||
|
||||
@@ -63,7 +63,10 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean {
|
||||
(fp.nativeQuestionIndex !== undefined && fp.nativeQuestionIndex !== 0)) return false;
|
||||
const q = call.questions[0]!;
|
||||
const title = q.question.split('\n')[0]!.trim();
|
||||
const issue = /^(?:D[1-9]\d*\s*[—–:-]\s*)?Issue ([1-9]\d*)(?: \((?:(?:G[1-9]\d*|Pass [1-7]), )?(?:Visual Hierarchy|Spacing|Color|Typography|Motion)\))?: ([^?]+)\?$/i.exec(title);
|
||||
// This primary-action decision can name the control in its Issue header.
|
||||
// F labels annotate findings; they do not establish review identity alone.
|
||||
const headerActionIssue = /^(?:D[1-9]\d*\s*[—–:-]\s*)?Issue ([1-9]\d*)(?: \(F[1-9]\d*\))?: (How should the header action group establish the primary action)\?$/i.exec(title);
|
||||
const issue = /^(?:D[1-9]\d*\s*[—–:-]\s*)?Issue ([1-9]\d*)(?: \((?:(?:G[1-9]\d*|Pass [1-7]), )?(?:Visual Hierarchy|Spacing|Color|Typography|Motion)\))?: ([^?]+)\?$/i.exec(title) ?? headerActionIssue;
|
||||
if (!issue || !new RegExp(`^Issue ${issue[1]}(?:: [A-Za-z][A-Za-z0-9 _-]{0,39})?$`, 'i').test(q.header.trim()) ||
|
||||
/<gstack-qid:/i.test(q.question) || q.multiSelect ||
|
||||
q.options.length < 2 || new Set(q.options.map(o => o.label)).size !== q.options.length ||
|
||||
@@ -73,13 +76,14 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean {
|
||||
// Reviewer participation or workflow navigation can also use Issue labels.
|
||||
if (!/\b(?:buttons?|primary(?: header)? actions?|hierarchy|spacing|contrast|colou?rs?|labels?|typography|fonts?|loading|spinner|skeleton|motion)\b/i.test(issue[2]!)) return false;
|
||||
const opposed = q.options.filter(o => /^(?:[1-9]\d*[A-Z](?:[).:]\s*|\s+))?(?:Defer|Decline|Leave|Keep|Accept the gap)\b/i.test(o.label));
|
||||
const repair = /\b(?:fix|resolve|address)\b/i.test(title) &&
|
||||
const repair = !headerActionIssue && /\b(?:fix|resolve|address)\b/i.test(title) &&
|
||||
q.options.some(o => /\b(?:closing|closes|fixes|resolves?|applies?)\b/i.test(o.description ?? ''));
|
||||
// A source citation alone can describe a report or the next reviewer.
|
||||
// Bind the alternate wording to a named control's concrete style amendment
|
||||
// and the opposed choice that leaves the documented violation unresolved.
|
||||
const primary = /^Make ([A-Za-z][A-Za-z0-9 _-]{0,39}) the (?:visible|visually|(?:only|single)(?: filled| visually)?) primary (?:header )?action(?: in the header)?$/i.exec(issue[2]!) ??
|
||||
/^How should (?:the )?(?:header )?actions establish that ([A-Za-z][A-Za-z0-9 _-]{0,39}) is the primary action$/i.exec(issue[2]!);
|
||||
/^How should (?:the )?(?:header )?actions establish that ([A-Za-z][A-Za-z0-9 _-]{0,39}) is the primary action$/i.exec(issue[2]!) ??
|
||||
(headerActionIssue && new RegExp(`^Issue ${issue[1]}: ([A-Za-z][A-Za-z0-9 _-]{0,39})$`, 'i').exec(q.header.trim()));
|
||||
const explicitStyle = primary && `${primary[1]} filled (?:primary )?#[0-9a-f]{6}(?:/| with )(?:white|black)(?: text)?; ` +
|
||||
'[A-Za-z][A-Za-z0-9 ,/_-]{0,99} neutral ghost(?: buttons)?\\.';
|
||||
const amendments = primary && [
|
||||
@@ -97,17 +101,26 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean {
|
||||
'[A-Za-z][A-Za-z0-9 ,/_-]{0,99}\\s*=\\s*neutral ghost(?: buttons?)?,? per DESIGN\\.md\\b', 'i'),
|
||||
];
|
||||
const primaryHeader = !q.header.includes(':') || q.header.split(':')[1]!.trim().toLowerCase() === primary?.[1]?.toLowerCase();
|
||||
const ownedStatus = (value: string, index: number, source: string) =>
|
||||
/^(?:withdrawn|superseded|resolved|closed|hypothetical|rejected|cancelled|canceled|not current|no longer current)$/i.test(value) &&
|
||||
/(?:^|[.!?;]\s+|\n)(?:Correction:\s*)?(?:(?:This (?:issue|finding|question|amendment|deferral|style|fix|remedy|choice|option|(?:DESIGN\.md |token )?(?:requirement|contract))|Issue [1-9]\d*) (?:is|was|has been)|(?:these|the|this) (?:tokens?|styles?|primary treatment) (?:are|is|were|was|have been|has been)) $/i.test(source.slice(0, index));
|
||||
// The style wordings share one owned decision: a current equal-weight gap,
|
||||
// a named control's DESIGN.md amendment, and a different choice retaining it.
|
||||
const currentText = (text: string) => text
|
||||
// A following status assertion remains current after a parenthesized effort
|
||||
// estimate. Preserve the estimate and expose its boundary to the same guards.
|
||||
const currentText = (text: string) => (headerActionIssue
|
||||
? text.replace(/(\(human: ~?[0-9]+(?:\.[0-9]+)?(?:h|min) \/ CC: ~?[0-9]+(?:\.[0-9]+)?(?:h|min)\))(?=\s+\S)/g, '$1.')
|
||||
: text)
|
||||
.replace(/```[\s\S]*?(?:```|$)|~~~[\s\S]*?(?:~~~|$)/g, '')
|
||||
.replace(/^(?:\s*>| {4}|\t).*$/gm, '')
|
||||
.replace(/`([^`]+)`/g, (_, body: string) => /\s/.test(body) ? '' : body)
|
||||
.replace(/`([^`]+)`/g, (_, body: string, index: number, source: string) =>
|
||||
headerActionIssue && ownedStatus(body, index, source) ? body : /\s/.test(body) ? '' : body)
|
||||
// A quoted status scalar remains a current assertion when its unquoted
|
||||
// subject names this decision; whole quoted historical prose stays absent.
|
||||
.replace(/"[^"\n]*"|“[^”\n]*”/g, (quoted: string, index: number, source: string) =>
|
||||
/^(?:withdrawn|superseded|resolved|closed|hypothetical|rejected|cancelled|canceled|not current|no longer current)$/i.test(quoted.slice(1, -1)) &&
|
||||
/(?:^|[.!?;]\s+|\n)(?:Correction:\s*)?(?:(?:This (?:issue|finding|question|amendment|deferral|style|fix|remedy|choice|option|(?:DESIGN\.md |token )?(?:requirement|contract))|Issue [1-9]\d*) (?:is|was|has been)|(?:these|the|this) (?:tokens?|styles?|primary treatment) (?:are|is|were|was|have been|has been)) $/i.test(source.slice(0, index))
|
||||
.replace(headerActionIssue
|
||||
? /"[^"\n]*"|“[^”\n]*”|(?<!\w)'[^'\n]*'(?!\w)|‘[^’\n]*’/g
|
||||
: /"[^"\n]*"|“[^”\n]*”/g, (quoted: string, index: number, source: string) =>
|
||||
ownedStatus(quoted.slice(1, -1), index, source)
|
||||
? quoted.slice(1, -1) : '').replace(/\*\*/g, '');
|
||||
const questionText = currentText(q.question);
|
||||
const assessments = [...questionText.matchAll(/^ELI10: (.+)$/gm)];
|
||||
@@ -119,7 +132,9 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean {
|
||||
!/^(?:Project\/branch\/task:|\[P[0-3]\])\s*(?:If|When|Unless|Provided|Assuming)\b/im.test(prefix.join('\n')) &&
|
||||
!sourceAssessment.test(prefix.join(' ')) && !sourceAssessment.test(assessments[0]![1]!)
|
||||
? assessments[0]![1]! : '';
|
||||
const primaryAssessment = primary && new RegExp(`^(?:Right now|Today) ${primary[1]}(?:, [A-Za-z][A-Za-z0-9 _-]{0,39})+(?:,? and [A-Za-z][A-Za-z0-9 _-]{0,39})? (?:(?:all )?look (?:the same|identical)|are (?:all )?(?:(?:two|three|four|five|six|seven|eight|nine|ten|[1-9]\\d*) )?identical buttons)\\b`, 'i').exec(assessment)?.[0];
|
||||
const headerPeers = primary && headerActionIssue && new RegExp(`^${primary[1]}, ([A-Za-z][A-Za-z0-9 _-]{0,39}(?:, [A-Za-z][A-Za-z0-9 _-]{0,39})*(?:,? and [A-Za-z][A-Za-z0-9 _-]{0,39})?) currently look (?:the same|identical)\\.`, 'i').exec(assessment);
|
||||
const primaryAssessment = headerActionIssue ? headerPeers?.[0] :
|
||||
primary && new RegExp(`^(?:Right now|Today) ${primary[1]}(?:, [A-Za-z][A-Za-z0-9 _-]{0,39})+(?:,? and [A-Za-z][A-Za-z0-9 _-]{0,39})? (?:(?:all )?look (?:the same|identical)|are (?:all )?(?:(?:two|three|four|five|six|seven|eight|nine|ten|[1-9]\\d*) )?identical buttons)\\b`, 'i').exec(assessment)?.[0];
|
||||
const premiseSentence = assessment.split(/[.!?](?:\s|$)/)[0] ?? '';
|
||||
const currentPrimary = !!primaryAssessment && !/\b(?:not|never|no longer)\b/i.test(primaryAssessment) &&
|
||||
!/\b(?:archived|historical|hypothetical|quoted|example|previous|earlier)\b/i.test(premiseSentence);
|
||||
@@ -127,25 +142,33 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean {
|
||||
// amendment names the existing component variants that implement it.
|
||||
const numberValue = (value: string) => /^\d+$/.test(value) ? Number(value) :
|
||||
['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten'].indexOf(value.toLowerCase());
|
||||
const otherControls = primary && primaryAssessment
|
||||
const controlNames = (text: string) => text.toLowerCase().split(/,\s*(?:and\s+)?|\s+and\s+/).map(s => s.trim()).sort();
|
||||
const headerControls = headerPeers ? controlNames(headerPeers[1]!) : [];
|
||||
const otherControls = headerActionIssue ? headerControls.length : primary && primaryAssessment
|
||||
? primaryAssessment.replace(new RegExp(`^(?:Right now|Today) ${primary[1]},\\s*`, 'i'), '')
|
||||
.replace(/\s+(?:(?:all )?look (?:the same|identical)|are (?:all )?(?:(?:two|three|four|five|six|seven|eight|nine|ten|[1-9]\d*) )?identical buttons)$/, '')
|
||||
.split(/,\s*(?:and\s+)?|\s+and\s+/).length : 0;
|
||||
const variantContract = primary && new RegExp(`(?:^|[.!?]\\s+)DESIGN\\.md already says ${primary[1]} is the only filled button ` +
|
||||
'\\(#[0-9a-f]{6} with (?:white|black) text(?:, about [0-9]+(?:\\.[0-9]+)?:1 contrast)?\\) and the other ' +
|
||||
'(two|three|four|five|six|seven|eight|nine|ten|[1-9]\\d*) are neutral ghost buttons\\.', 'i').exec(assessment);
|
||||
const invalidContract = /(?:^|[.!?]\s+|\n)(?:Correction:\s*)?(?:this|that|the) (?:(?:DESIGN\.md|token) )?(?:requirement|contract) (?:is|was|has been) (?:withdrawn|superseded|rejected|cancelled|canceled|not current|no longer current)\b/i;
|
||||
const statusBoundary = headerActionIssue ? '[.!?;]' : '[.!?]';
|
||||
const invalidContract = new RegExp(`(?:^|${statusBoundary}\\s+|\\n)(?:Correction:\\s*)?(?:this|that|the) (?:(?:DESIGN\\.md|token) )?(?:requirement|contract) (?:is|was|has been) (?:withdrawn|superseded|rejected|cancelled|canceled|not current|no longer current)\\b`, 'i');
|
||||
const namedContract = primary && new RegExp(`(?:^|[.!?]\\s+)DESIGN\\.md already says ${primary[1]} is the only filled primary button and the other (two|three|four|five|six|seven|eight|nine|ten|[1-9]\\d*) are neutral ghost buttons\\.`, 'i').exec(assessment);
|
||||
const headerContract = primary && headerActionIssue && new RegExp(`(?:^|[.!?]\\s+)DESIGN\\.md already answers it: ${primary[1]} is the only filled primary button, the other (two|three|four|five|six|seven|eight|nine|ten|[1-9]\\d*) are neutral ghost buttons\\.`, 'i').exec(assessment);
|
||||
const conditionalHeader = (text: string) => /(?:^|[.!?;]\s+|\n)(?:[✅❌]\s*)?(?:Correction:\s*)?(?:If|When|Unless|Assuming|Provided)\b/i.test(text);
|
||||
const currentHeaderContract = !headerActionIssue || (headerContract && headerControls.length > 0 &&
|
||||
new Set(headerControls).size === headerControls.length && !headerControls.includes(primary![1]!.toLowerCase()) &&
|
||||
numberValue(headerContract[1]!) === headerControls.length && !conditionalHeader(questionText));
|
||||
const statedVariant = (variantContract || namedContract) &&
|
||||
numberValue((variantContract || namedContract)![1]!) === otherControls &&
|
||||
!/\b(?:proposed|hypothetical|quoted|historical|source)\s+(?:example|contract|requirement)\b/i.test(assessment.slice(0, (variantContract || namedContract)!.index)) &&
|
||||
!invalidContract.test(questionText);
|
||||
const withdrawn = new RegExp(`(?:^|[.!?]\\s+|\\n)(?:Correction:\\s*)?(?:(?:This (?:issue|finding|question|amendment|deferral|style|fix|remedy|choice|option)|Issue ${issue[1]}) (?:is|was|has been) (?:withdrawn|superseded|resolved|closed|hypothetical|rejected|cancelled|canceled|not current|no longer current)|We have (?:resolved|closed|withdrawn) this (?:issue|finding)|No current (?:issue|finding|gap|violation) (?:remains|exists))\\b`, 'i');
|
||||
const withdrawn = new RegExp(`(?:^|${statusBoundary}\\s+|\\n)(?:Correction:\\s*)?(?:(?:This (?:issue|finding|question|amendment|deferral|style|fix|remedy|choice|option)|Issue ${issue[1]}) (?:is|was|has been) (?:withdrawn|superseded|resolved|closed|hypothetical|rejected|cancelled|canceled|not current|no longer current)|We have (?:resolved|closed|withdrawn) this (?:issue|finding)|No current (?:issue|finding|gap|violation) (?:remains|exists))\\b`, 'i');
|
||||
const closedGap = /(?:^|[.!?;]\s+|\n)(?:Correction:\s*)?(?:this|the|that) (?:gap|violation) (?:is|was|has been) (?:already\s+|now\s+)?(?:resolved|fixed|closed)\b/i;
|
||||
const cancelledStyle = /(?:^|[.!?;]\s+|\n)(?:Correction:\s*)?(?:do not|don't|never|skip|cancel|withdraw)\s+(?:apply|use|add|keep)\s+(?:(?:these|the|this)\s+)?(?:tokens?|styles?|primary treatment)\b/i;
|
||||
const withdrawnStyles = /(?:^|[.!?;]\s+|\n)(?:Correction:\s*)?(?:these|the|this) (?:tokens?|styles?|primary treatment) (?:are|is|were|was|have been|has been) (?:withdrawn|rejected|cancelled|canceled|not current|no longer current)\b/i;
|
||||
const choiceIds = q.options.map(o => /^([1-9]\d*)[A-Z](?:[).:]?\s+)/.exec(o.label));
|
||||
const primaryRepair = primaryHeader && amendments && currentPrimary &&
|
||||
const primaryRepair = primaryHeader && amendments && currentPrimary && currentHeaderContract &&
|
||||
prefix.filter(line => /^Project\/branch\/task:/.test(line)).length === 1 &&
|
||||
!!call.answeredAt && Number.isFinite(Date.parse(call.answeredAt)) &&
|
||||
choiceIds.every(id => id?.[1] === issue[1]) &&
|
||||
@@ -153,14 +176,17 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean {
|
||||
!withdrawn.test(questionText) && !closedGap.test(questionText) && !withdrawnStyles.test(questionText) && !invalidContract.test(questionText) &&
|
||||
q.options.some(amendment => {
|
||||
const body = currentText(amendment.description ?? '');
|
||||
const style = amendments.map(pattern => pattern.exec(body)).find(Boolean)?.[0];
|
||||
const headerStyle = primary && headerActionIssue && new RegExp(`^(?:✅\\s*)?${primary[1]} becomes the only filled #[0-9a-f]{6} button with (?:white|black) text; ([A-Za-z][A-Za-z0-9 ,_-]{0,119}) become neutral ghost buttons, exactly as DESIGN\\.md states\\.`, 'i').exec(body);
|
||||
const style = headerActionIssue ? headerStyle?.[0] : amendments.map(pattern => pattern.exec(body)).find(Boolean)?.[0];
|
||||
if (headerActionIssue && (!headerStyle || conditionalHeader(body) || invalidContract.test(body) ||
|
||||
JSON.stringify(controlNames(headerStyle[1]!)) !== JSON.stringify(headerControls))) return false;
|
||||
const variantLine = /^✅\s*Uses the existing Button primary and ghost variants from DESIGN\.md; no new styles\./m.exec(body);
|
||||
const benefits = variantLine ? body.slice(0, variantLine.index).trim().split('\n').filter(Boolean) : [];
|
||||
const labelledRoles = /^✅ Matches DESIGN\.md exactly: one filled primary, (two|three|four|five|six|seven|eight|nine|ten|[1-9]\d*) neutral ghosts, [1-9]\d*px targets kept\./.exec(body);
|
||||
const labelledStyle = statedVariant && namedContract && labelledRoles &&
|
||||
numberValue(labelledRoles[1]!) === otherControls &&
|
||||
new RegExp(`^[1-9]\\d*[A-Z]\\) ${primary![1]} filled #[0-9a-f]{6}/(?:white|black), others ghost(?: \\(recommended\\))?$`, 'i').test(amendment.label);
|
||||
const variantRepair = (labelledStyle || (statedVariant && variantContract && variantLine &&
|
||||
const variantRepair = !headerActionIssue && (labelledStyle || (statedVariant && variantContract && variantLine &&
|
||||
new RegExp(`^[1-9]\\d*[A-Z] Filled ${primary![1]}, ghost others(?: \\(recommended\\))?$`, 'i').test(amendment.label) &&
|
||||
benefits.every(line => /^✅\s*(?!(?:If|When|Unless|Historical|Hypothetical|Quoted|Source|Example)\b)\S/i.test(line)) &&
|
||||
!/\b(?:archived|historical|hypothetical|quoted|previous|earlier)\b/i.test(benefits.join(' ')))) &&
|
||||
@@ -182,6 +208,13 @@ function ordinaryDesignIssue(fp: AskUserQuestionFingerprint): boolean {
|
||||
const remaining = pros && !sourceAssessment.test(pros[0]) ? deferralBody.slice(pros[0].length) : deferralBody;
|
||||
const retainedButtons = /^(?:❌\s*)?Keep all (two|three|four|five|six|seven|eight|nine|ten|[1-9]\d*) (?:header )?buttons identical; gap stays documented\./i.exec(remaining);
|
||||
const cancelledRetainedButtons = /(?:^|[.!?;]\s+|\n)(?:Correction:\s*)?(?:do not|don't|never|skip|cancel|withdraw) (?:keep|leave) (?:all )?(two|three|four|five|six|seven|eight|nine|ten|[1-9]\d*) (?:header )?buttons identical\b/i.exec(declined);
|
||||
if (headerActionIssue) {
|
||||
const keep = /^[1-9]\d*[A-Z]: Keep all (two|three|four|five|six|seven|eight|nine|ten|[1-9]\d*) identical(?: \(recommended\))?$/i.exec(defer.label);
|
||||
return defer !== amendment && keep && numberValue(keep[1]!) === headerControls.length + 1 &&
|
||||
!conditionalHeader(declined) && !sourceAssessment.test(declined) && !withdrawn.test(declined) && !invalidContract.test(declined) &&
|
||||
!closedGap.test(declined) && !cancelledRetainedButtons &&
|
||||
/^(?:❌\s*)?Ships a (?:known|documented) DESIGN\.md violation; the review score stays capped and users keep scanning a flat row\./i.test(remaining);
|
||||
}
|
||||
return defer !== amendment && !sourceAssessment.test(declined) && !withdrawn.test(declined) && !closedGap.test(declined) && !cancelledHeaderDeferral &&
|
||||
((retainedButtons && numberValue(retainedButtons[1]!) === otherControls + 1 &&
|
||||
(!cancelledRetainedButtons || numberValue(cancelledRetainedButtons[1]!) !== otherControls + 1)) ||
|
||||
|
||||
@@ -26,7 +26,7 @@ function explainedReversedSignatures(q: NativePlanQuestion, title: string): bool
|
||||
};
|
||||
const current = currentProse(lines.slice(explanation).join('\n'));
|
||||
if ((current.match(/^ELI10:/gm)?.length ?? 0) !== 1) return false;
|
||||
if (/(?:^|[.!?\n]\s*)(?:Correction:\s*)?(?:(?:this|that|the) (?:finding|explanation)|(?:(?:this|that|the) )?argument[- ]order (?:issue|defect)|these signatures)\b[^.\n]*\b(?:withdrawn|rejected|(?:already )?(?:fixed|resolved)|historical|not current)\b/i.test(current) ||
|
||||
if (/(?:^|[.!?\n]\s*)(?:Correction:\s*)?(?:(?:this|that|the) (?:finding|explanation)|(?:(?:this|that|the) )?argument[- ]order (?:issue|defect)|these signatures)\b[^.\n]*\b(?:withdrawn|rejected|(?:already )?(?:fixed|resolved)|historical|(?:not|no longer) current)\b/i.test(current) ||
|
||||
/(?:^|[.!?\n]\s*)(?:Correction:\s*)?(?:there is|there's) no argument[- ]order (?:issue|defect)\b/i.test(current) ||
|
||||
/(?:^|[.!?\n]\s*)(?:Correction:\s*)?run_eval and run_batch now (?:use|take) the same positional order\b/i.test(current)) return false;
|
||||
// The same offered action must align both functions and retain the call-site
|
||||
@@ -43,55 +43,84 @@ function decisionGaps(q: NativePlanQuestion): DevexSeededGap[] {
|
||||
const questionMarks = title.match(/\?/g)?.length ?? 0;
|
||||
// Journey labels, possessives and a positive inclusive aside format the
|
||||
// asserted subject. Keep the original title for all meaning/currentness checks.
|
||||
const stage = title.match(/^Journey stage (?:HELLO WORLD|REAL USAGE|DEBUG|UPGRADE): (.+)$/i);
|
||||
// These six stages come from the skill's journey trace. A decision may span
|
||||
// adjacent touchpoints without changing the subject or who asserts it.
|
||||
const journeyStage = '(?:DISCOVER|INSTALL|HELLO WORLD|REAL USAGE|DEBUG|UPGRADE)';
|
||||
const stage = title.match(new RegExp(`^Journey stage ${journeyStage}(?:\\s*\\/\\s*${journeyStage})?: (.+)$`, 'i'));
|
||||
if (stage && /^(?:Assuming|Provided)\b/i.test(stage[1]!.trim())) return [];
|
||||
const assertionTitle = stage ? stage[1]!
|
||||
.replace(/\b([A-Za-z0-9_.]+)['’]s\b/g, '$1')
|
||||
.replace(/, including ([A-Za-z0-9_-]+(?: [A-Za-z0-9_-]+){0,6}),/gi, (aside, subject: string) =>
|
||||
/\b(?:if|unless|assuming|provided|except|excluding|only|no|not|never|without|was|were|is|are|has|had|may|might|could|would|historical|earlier|quoted|source|example|hypothetical|fixed|resolved|cancelled|canceled|withdrawn|rejected|superseded)\b/i.test(subject) ? aside : '')
|
||||
: title;
|
||||
// A native choice can present an asserted defect as its title. Punctuation
|
||||
// does not turn its concrete offered repairs into a missing review decision.
|
||||
const declaration = questionMarks === 0 &&
|
||||
/^(?:[A-Za-z0-9_.]+\s+){1,12}(?:points?|references?|blocks?|requires?|takes?|raises?|removes?|drops?)\b/i.test(assertionTitle) &&
|
||||
// A defect heading can assert a prerequisite or compare named signatures
|
||||
// without a finite verb. Keep these semantic families narrow: a topic label,
|
||||
// healthy signature pair or optional check is not an asserted defect.
|
||||
const nominalDefect = /^(?:Mandatory|Required) (?:\d+(?:\.\d+)?[- ](?:minute|second) )?(?:remote )?CI (?:check|gate) before (?:the )?first local (?:result|evaluation|run)[.?]?$/i.test(assertionTitle) ||
|
||||
/^run_eval\(\s*dataset\s*,\s*evaluator\s*\) (?:vs\.?|versus|and) run_batch\(\s*evaluator\s*,\s*dataset\s*\): (?:reversed|opposite|swapped) (?:positional|argument) order[.?]?$/i.test(assertionTitle);
|
||||
const nominalSubject = /^(?:Mandatory|Required|Optional)\b[^?!\n]*\bCI (?:check|gate)\b/i.test(assertionTitle) ||
|
||||
/^run_eval\([^)]+\) (?:vs\.?|versus|and) run_batch\([^)]+\):/i.test(assertionTitle);
|
||||
if (nominalSubject && !nominalDefect) return [];
|
||||
const signatureDeclaration = /^run_eval\(\s*dataset\s*,\s*evaluator\s*\) and run_batch\(\s*evaluator\s*,\s*dataset\s*\) (?:take|takes)\b/i.test(assertionTitle);
|
||||
if (/^run_eval\([^)]+\) and run_batch\([^)]+\) (?:take|takes)\b/i.test(assertionTitle) && !signatureDeclaration) return [];
|
||||
const finiteTitle = signatureDeclaration ? assertionTitle.replace(/\([^)]*\)/g, '') : assertionTitle;
|
||||
// Negative availability asserts a missing referenced file. Bind it to that
|
||||
// object; do not erase a negation of the quickstart's own reference or gate.
|
||||
const absentReference = /\b(?:points?|references?) (?:at|to) (?:examples\/first_eval\.py|(?:a|the) (?:file|example)),? (?:which|that) (?:is not in (?:the )?(?:package|wheel)(?: or (?:the )?(?:release )?examples archive)?|does not (?:ship|exist))[.?]?$/i.test(assertionTitle);
|
||||
const newAssertion = nominalDefect || signatureDeclaration || absentReference;
|
||||
const guardedDeclaration = Boolean(stage || newAssertion);
|
||||
const polarityTitle = absentReference ? title.replace(/\bdoes not (ship|exist)([.?]?)$/i, 'is absent$2') : title;
|
||||
// Punctuation cannot route a newly admitted asserted family around its
|
||||
// ownership checks; an offered alternate still resolves the same decision.
|
||||
const declaration = (questionMarks === 0 || (questionMarks === 1 && title.endsWith('?'))) &&
|
||||
(/^(?:[A-Za-z0-9_.]+\s+){1,12}(?:points?|references?|blocks?|requires?|takes?|raises?|removes?|drops?)\b/i.test(finiteTitle) || nominalDefect) &&
|
||||
!/^`[^`]*`$/.test(rawTitle) &&
|
||||
!/\b(?:if|unless|suppose|might|may|could|would|previously|earlier|historical|hypothetical|example|quoted|source|never|no longer|does not|do not|did not)\b/i.test(title);
|
||||
!/\b(?:if|unless|suppose|might|may|could|would|previously|earlier|historical|hypothetical|example|quoted|source|never|no longer|does not|do not|did not)\b/i.test(polarityTitle);
|
||||
if (newAssertion && !declaration) return [];
|
||||
if ((!declaration && (!title.endsWith('?') || questionMarks !== 1)) ||
|
||||
/^(?:>|"|“|Example\b|Quoted\b|Suppose\b)|\bhypothetical\b/i.test(title) ||
|
||||
/^`[^`]*`[.?]?$/.test(rawTitle) ||
|
||||
/^(?:>|"|“|Example\b|Quoted\b|Source(?: excerpt| example)?[,:.]|Historical\b|Earlier review\b|If (?:approved|accepted)\b|Assuming\b|Provided\b|Suppose\b)|\bhypothetical\b/i.test(title) ||
|
||||
/\b(?:if|once|when|unless) (?:approved|accepted)|\b(?:after|pending) approval\b/i.test(title) ||
|
||||
/\b(?:continue|proceed|next section|move on|format|already (?:fixed|resolved))\b/i.test(title) ||
|
||||
/\b(?:have|did)\b[^?]*\bread\b|\b(?:narrative|trace|recap|summary)\b[^?]*\b(?:accurate|match|confirm)\b/i.test(title) ||
|
||||
/\b(?:report|summary|recap)\b[^?]*\b(?:mention|include|reference|list)\b|\b(?:mention|include|reference|list)\b[^?]*\b(?:report|summary|recap)\b/i.test(title)) return [];
|
||||
let offered = q.options;
|
||||
if (declaration) {
|
||||
const currentProse = (text: string) => {
|
||||
const currentProse = (text: string, offeredAction = false) => {
|
||||
// An option's trailing effort estimate separates its prose from an owned
|
||||
// status even without punctuation. Keep it on the same line so a quoted
|
||||
// historical sentence is still removed as one quotation below.
|
||||
const bounded = guardedDeclaration && offeredAction ? text.replace(/(\(human:[^()\n]{1,80}\/ CC:[^()\n]{1,80}\))[ \t]+(?=(?:Correction:\s*)?(?:(?:this|that|the) (?:option|action|correction)|D\s*[1-9]\d*) (?:is|was|has been) ["“'‘`]?(?:cancelled|canceled|superseded|withdrawn|rejected|(?:not|no longer) current)\b)/gi, '$1. ') : text;
|
||||
// Preserve a scalar status asserted by a current, unquoted owner before
|
||||
// removing source quotations. This applies only to the new stage form.
|
||||
const owned = stage ? text.replace(/(^|[.!?\n]\s*)((?:Correction:\s*)?(?:(?:this|that|the) (?:finding|issue|gap|defect|explanation|option|action|correction)|D\s*[1-9]\d*) (?:is|was|has been) )["“](cancelled|canceled|superseded|withdrawn|rejected|not current)["”]/gim, '$1$2$3') : text;
|
||||
// removing source quotations. The owner must still match this decision.
|
||||
const owned = guardedDeclaration ? bounded.replace(/(^|[.!?\n]\s*)((?:Correction:\s*)?(?:(?:this|that|the) (?:finding|issue|gap|defect|explanation|option|action|correction)|D\s*[1-9]\d*) (?:is|was|has been) )["“'‘`](cancelled|canceled|superseded|withdrawn|rejected|(?:not|no longer) current)["”'’`]/gim, '$1$2$3') : bounded;
|
||||
let fence = false;
|
||||
return owned.split('\n').filter(line => {
|
||||
if (/^\s*(?:```|~~~)/.test(line)) { fence = !fence; return false; }
|
||||
return !fence && !/^\s*>/.test(line);
|
||||
}).join('\n')
|
||||
.replace(/(^|[.!?\n]\s*)((?:Correction:\s*)?(?:this|that|the) (?:finding|issue|gap|defect|explanation|option|action|correction) (?:is|was|has been) )["“](withdrawn|rejected|(?:already )?(?:fixed|resolved)|historical|not current|cancelled)["”]/gim, '$1$2$3')
|
||||
.replace(/(^|[.!?\n]\s*)((?:Correction:\s*)?(?:this|that|the) (?:finding|issue|gap|defect|explanation|option|action|correction) (?:is|was|has been) )["“](withdrawn|rejected|(?:already )?(?:fixed|resolved)|historical|(?:not|no longer) current|cancelled)["”]/gim, '$1$2$3')
|
||||
.replace(/`[^`\n]*`|"[^"\n]*"|“[^”\n]*”/g, '');
|
||||
};
|
||||
const sourceFrame = /(?:^|[.!?\n;]\s*)(?:(?:ELI10|Project\/branch\/task):\s*)?(?:(?:Source(?: excerpt| example)?|Quoted(?: source| example)?|Historical(?: example| assessment)?(?: only)?|Earlier(?: review)? assessment|Example|Hypothetical(?: example| assessment)?|If approved|If accepted)[,:]|(?:The following|This assessment|This explanation)\b[^.\n]*\b(?:quoted|source|historical|hypothetical|example)\b|Historically,)/i;
|
||||
const sourceFrame = /(?:^|[.!?\n;]\s*)(?:(?:ELI10|Project\/branch\/task):\s*)?(?:(?:Source(?: excerpt| example)?|Quoted(?: source| example)?|Historical(?: example| assessment)?(?: only)?|Earlier(?: review)? assessment|Example|Hypothetical(?: example| assessment| scenario)?|If approved|If accepted)[,:.]|(?:The following|This assessment|This explanation)\b[^.\n]*\b(?:quoted|source|historical|hypothetical|example)\b|Historically,)/i;
|
||||
const lines = q.question.split('\n'), explanation = lines.findIndex(line => /^ELI10:/.test(line));
|
||||
const preface = lines.slice(0, explanation < 0 ? undefined : explanation + 1).join('\n');
|
||||
if (stage && /^(?:Project\/branch\/task|ELI10):\s*(?:Assuming|Provided)\b/im.test(currentProse(preface))) return [];
|
||||
if (guardedDeclaration && /^(?:Project\/branch\/task|ELI10):\s*(?:Assuming|Provided)\b/im.test(currentProse(preface))) return [];
|
||||
if (/^\s*(?:```|~~~)/m.test(preface) || sourceFrame.test(currentProse(preface)) ||
|
||||
/\bnot (?:a )?current (?:finding|issue|defect)\b/i.test(currentProse(preface))) return [];
|
||||
const current = currentProse(q.question);
|
||||
const decision = stage && /^D\s*([1-9]\d*)\s*[—–:-]/i.exec(q.question);
|
||||
if (decision && new RegExp(`(?:^|[.!?\\n]\\s*)(?:Correction:\\s*)?D\\s*${decision[1]} (?:is|was|has been) (?:withdrawn|rejected|cancelled|canceled|superseded|not current)\\b`, 'i').test(current)) return [];
|
||||
if (stage && /(?:^|[.!?\n]\s*)(?:Correction:\s*)?(?:this|that|the) (?:finding|issue|gap|defect|explanation) (?:is|was|has been) (?:cancelled|canceled|superseded)\b/i.test(current)) return [];
|
||||
if (/(?:^|[.!?\n]\s*)(?:Correction:\s*)?(?:(?:this|that|the) (?:finding|issue|gap|defect|explanation) (?:is|was|has been) (?:withdrawn|rejected|(?:already )?(?:fixed|resolved)|historical|not current|(?:a |only a )?source example)|there is no (?:current )?(?:finding|issue|gap|defect))\b/i.test(current)) return [];
|
||||
const decision = guardedDeclaration && /^D\s*([1-9]\d*)\s*[—–:-]/i.exec(q.question);
|
||||
if (decision && new RegExp(`(?:^|[.!?\\n]\\s*)(?:Correction:\\s*)?D\\s*${decision[1]} (?:is|was|has been) (?:withdrawn|rejected|cancelled|canceled|superseded|(?:not|no longer) current)\\b`, 'i').test(current)) return [];
|
||||
if (guardedDeclaration && /(?:^|[.!?\n]\s*)(?:Correction:\s*)?(?:this|that|the) (?:finding|issue|gap|defect|explanation) (?:is|was|has been) (?:cancelled|canceled|superseded)\b/i.test(current)) return [];
|
||||
if (/(?:^|[.!?\n]\s*)(?:Correction:\s*)?(?:(?:this|that|the) (?:finding|issue|gap|defect|explanation) (?:is|was|has been) (?:withdrawn|rejected|(?:already )?(?:fixed|resolved)|historical|(?:not|no longer) current|(?:a |only a )?source example)|there is no (?:current )?(?:finding|issue|gap|defect))\b/i.test(current)) return [];
|
||||
// A declaration's action evidence must belong to a current offered option,
|
||||
// rather than an example or an explicitly withdrawn correction.
|
||||
const action = (text: string) => currentProse(text.replace(/`([A-Za-z_$][\w.$/-]*(?:\([^`\n]*\))?)`/g, '$1'));
|
||||
const action = (text: string) => currentProse(text.replace(/`([A-Za-z_$][\w.$/-]*(?:\([^`\n]*\))?)`/g, '$1'), true);
|
||||
offered = offered.filter(option => {
|
||||
const text = `${option.label}\n${option.description ?? ''}`, prose = currentProse(text);
|
||||
if (stage && /(?:^|[.!?\n]\s*)(?:Correction:\s*)?(?:this|that|the) (?:option|action|correction) (?:is|was|has been) (?:canceled|superseded)\b/i.test(prose)) return false;
|
||||
const text = `${option.label}\n${option.description ?? ''}`, prose = currentProse(text, true);
|
||||
if (guardedDeclaration && /(?:^|[.!?\n]\s*)(?:Correction:\s*)?(?:this|that|the) (?:option|action|correction) (?:is|was|has been) (?:cancelled|canceled|superseded|withdrawn|rejected|(?:not|no longer) current)\b/i.test(prose)) return false;
|
||||
if (guardedDeclaration && /^(?:Assuming|Provided)\b/im.test(prose)) return false;
|
||||
if (decision && new RegExp(`(?:^|[.!?\\n]\\s*)(?:Correction:\\s*)?D\\s*${decision[1]} (?:is|was|has been) (?:withdrawn|rejected|cancelled|canceled|superseded|(?:not|no longer) current)\\b`, 'i').test(prose)) return false;
|
||||
return !/^(?:>|"|“)|^`[^`]*`$/.test(option.label.trim()) && !sourceFrame.test(prose) &&
|
||||
!/(?:^|[.!?\n]\s*)(?:Correction:\s*)?(?:this|the) (?:option|action|correction) (?:is|was|has been) (?:withdrawn|rejected|cancelled)\b/i.test(prose);
|
||||
}).map(option => ({ ...option, label: action(option.label), description: action(option.description ?? '') }));
|
||||
@@ -110,7 +139,7 @@ function decisionGaps(q: NativePlanQuestion): DevexSeededGap[] {
|
||||
(options.some(o => /\b(?:no CI gate|remove|move|skip|bypass|gate)\b/i.test(o) && /\b(?:CI|check|gate|local|demo)\b/i.test(o)) || directAction('remove|move|skip|bypass|gate'))) found.push('local-ci-gate');
|
||||
if (/\bquickstart\b|examples\/first_eval\.py/i.test(title) &&
|
||||
/\b(?:README|file|example|demo|missing|absent|package|wheel|ship|point)\b|first_eval\.py/i.test(title) &&
|
||||
(!declaration || /\b(?:not (?:shipped|included|available|present)|missing|absent|nonexistent|does not exist)\b/i.test(title)) &&
|
||||
(!declaration || absentReference || /\b(?:not (?:shipped|included|available|present)|missing|absent|nonexistent|does not exist)\b/i.test(title)) &&
|
||||
(options.some(o => /\b(?:point|ship|add|demo is)\b/i.test(o) && /\bquickstart\b|first_eval\.py/i.test(o)) || directAction('point|ship|add|replace|fix'))) found.push('missing-quickstart');
|
||||
if (explainedReversedSignatures(q, title) || (/\brun_eval\b/i.test(title) && /\brun_batch\b/i.test(title) &&
|
||||
/\b(?:arguments?|order|positional|reversed|opposite|consistent|align|unify|dataset|evaluator)\b/i.test(title) &&
|
||||
@@ -119,7 +148,7 @@ function decisionGaps(q: NativePlanQuestion): DevexSeededGap[] {
|
||||
if (/\bAuthError\b|\binvalid API key\b/i.test(title) &&
|
||||
/\b(?:error|message|code|cause|fix|guidance|opaque|explain)\b|request failed/i.test(title) &&
|
||||
(!declaration || /\b(?:no (?:cause|fix|explanation|code)|opaque)\b|request failed/i.test(title)) &&
|
||||
(options.some(o => /\bcodes?\b/i.test(o) && /\b(?:cause|fix|link)\b/i.test(o)) || directAction('add|include|explain|replace|report|give'))) found.push('opaque-auth-error');
|
||||
(options.some(o => (/\bcodes?\b/i.test(o) || /^(?:[A-D]\)\s*)?Coded\b/i.test(o)) && /\b(?:cause|fix|link)\b/i.test(o)) || directAction('add|include|explain|replace|report|give'))) found.push('opaque-auth-error');
|
||||
if (/Client\.evaluate\b/i.test(title) &&
|
||||
/Client\.run\b|\b(?:v\d+|version \d+|alias|deprecation|migration)\b/i.test(title) &&
|
||||
/\b(?:alias|warning|compatibility|deprecat\w*|migration|remov\w*|rename|keep)\b/i.test(title) &&
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import type { NativePlanQuestion } from './plan-count-transcript';
|
||||
|
||||
/** A current cache-writer decision may name its actors in the plan context.
|
||||
* Called only after engNumberedFindingAUQ validates completed native metadata. */
|
||||
export function engCacheWriterDecision(q: NativePlanQuestion): boolean {
|
||||
const lines = q.question.split('\n');
|
||||
const ordinal = /^D([1-9]\d*) [—–:-] Who is allowed to write to the auth cache\?$/.exec(lines[0] ?? '')?.[1];
|
||||
if (!ordinal || q.header !== 'Cache writes') return false;
|
||||
const context = /^Project\/branch\/task: (\S[^\n]*), ([A-Za-z_$][\w$]*) and ([A-Za-z_$][\w$]*) both mutating one backing cache \(([\w./-]+\.md):\d+(?:, \d+(?:[-–]\d+)?)?\)\.$/.exec(lines[1] ?? '');
|
||||
const assessment = /^ELI10: Two services write to the same cache and nothing orders their writes\. /.test(lines[2] ?? '');
|
||||
if (!context || context[2] === context[3] || !assessment ||
|
||||
lines.filter(line => /^Project\/branch\/task:/.test(line)).length !== 1 ||
|
||||
lines.filter(line => /^ELI10:/.test(line)).length !== 1) return false;
|
||||
const boundary = '(?:^|[.!?;]\\s+|\\n|[✅❌]\\s*)(?:Correction:\\s*)?';
|
||||
const owner = `(?:(?:this|the|that) (?:finding|issue|gap|assessment|option|action|remedy|race|single-writer requirement)|D\\s*${ordinal})`;
|
||||
const status = '(?:withdrawn|superseded|rejected|cancelled|canceled|resolved|closed|hypothetical|not current|no longer current)';
|
||||
const current = (text: string) => text
|
||||
.replace(/```[\s\S]*?(?:```|$)|~~~[\s\S]*?(?:~~~|$)/g, '')
|
||||
.replace(/^(?:\s*>| {4}|\t).*$/gm, '')
|
||||
.replace(new RegExp(`(${boundary}${owner} (?:is|was|has been) )["“'‘\x60](${status})["”'’\x60]`, 'gim'), '$1$2')
|
||||
.replace(/"[^"\n]*"|“[^”\n]*”|(?<!\w)'[^'\n]*'(?!\w)|‘[^’\n]*’/g, '')
|
||||
.replace(/`([^`\n]*)`/g, (_, value: string) => /^[A-Za-z_$][\w$]*$/.test(value) ? value : '');
|
||||
const framed = new RegExp(`${boundary}(?:Source(?: excerpt| example)?|Quoted(?: source| example)?|Historical(?: assessment| example)?|Hypothetical(?: scenario| example)?|Earlier review)(?:[.:,]|\\s)|${boundary}(?:if|unless|when|assuming|provided|suppose|imagine)\\b`, 'i');
|
||||
const closed = new RegExp(`${boundary}${owner} (?:is|was|has been) ${status}\\b|${boundary}(?:there is )?no current (?:gap|risk|finding|race) (?:remains|exists)\\b`, 'i');
|
||||
// The first assertion is current; its following revoked-token scenario is
|
||||
// a causal explanation, not a condition on whether this review occurs.
|
||||
if (/\b(?:if|when|once|unless) (?:approved|accepted)|\b(?:after|pending) approval\b/i.test(current(context[1]!)) || framed.test(current(context[1]!)) || framed.test(current(lines.slice(0, 3).join('\n').split('ELI10:')[0]!)) ||
|
||||
closed.test(current(q.question))) return false;
|
||||
const escape = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const actors = [context[2]!, context[3]!];
|
||||
const contradiction = new RegExp(`${boundary}(?:(?:the|these|both) services (?:no longer (?:writes?|mutates?)|now serialize)|(?:${actors.map(escape).join('|')}) (?:no longer|does not) (?:writes?|mutates?)|(?:the |this )?(?:auth )?cache (?:is no longer shared|now serializes)|(?:the )?(?:race is (?:resolved|closed)|(?:writes|writers) are (?:now )?(?:ordered|serialized)))\\b`, 'i');
|
||||
if (contradiction.test(current(q.question))) return false;
|
||||
const rows = q.options.map(option => ({
|
||||
id: new RegExp(`^${ordinal}([A-D]) (.+?)(?: \\(recommended\\))?$`).exec(option.label),
|
||||
text: current(option.description ?? '').trim(),
|
||||
}));
|
||||
if (rows.some(row => !row.id) || new Set(rows.map(row => row.id![1])).size !== rows.length) return false;
|
||||
const cancelled = new RegExp(`${boundary}(?:do not|don't|never|skip|cancel|withdraw) (?:use|keep|accept|adopt|choose|proceed|reject|write|serialize|document)\\b`, 'i');
|
||||
if (rows.some(row => framed.test(row.text) || closed.test(row.text) || cancelled.test(row.text))) return false;
|
||||
const remedy = rows.find(row => row.id![2] === 'Single writer + version');
|
||||
const unchanged = rows.find(row => row.id![2] === 'Accept the race');
|
||||
if (!remedy || !unchanged) return false;
|
||||
const writers = /^([A-Za-z_$][\w$]*) writes with policy-version tag; adapter rejects stale writes; ([A-Za-z_$][\w$]*) reads\/invalidates\./.exec(remedy.text);
|
||||
if (!writers || writers[1] === writers[2] || !actors.includes(writers[1]!) || !actors.includes(writers[2]!)) return false;
|
||||
const override = new RegExp(`${boundary}(?:${escape(writers[2]!)} (?:also |still )?writes|${escape(writers[1]!)} (?:does not|no longer) writes|(?:the )?adapter (?:accepts stale writes|does not reject stale writes)|(?:the )?(?:version check|single-writer requirement) is (?:removed|disabled|optional))\\b`, 'i');
|
||||
const unchangedOverride = new RegExp(`${boundary}(?:only (?:${actors.map(escape).join('|')}) writes|(?:the |both )?writers no longer (?:write|mutate)|(?:the )?race is (?:no longer current|resolved|closed))\\b`, 'i');
|
||||
if (override.test(remedy.text) || !/^Keep both writers as planned and document the known race\./.test(unchanged.text) || contradiction.test(unchanged.text) || unchangedOverride.test(unchanged.text)) return false;
|
||||
return rows.every(row => row === remedy || row === unchanged ||
|
||||
(row.id![2] === 'Single writer only' &&
|
||||
new RegExp(`^${escape(writers[1]!)} writes, ${escape(writers[2]!)} reads and invalidates\\. No version check\\.`).test(row.text) && !override.test(row.text)));
|
||||
}
|
||||
@@ -25,6 +25,48 @@ function seedSubjects(q: NativePlanQuestionCall['questions'][number]): Seed[] {
|
||||
const subject = q.question.split('\n').find(line => line.trim())?.trim() ?? '';
|
||||
if (/^(?:>|`{3}|~{3}|example\b|quote\b|["“])|\b(?:no (?:issue|defect)|already (?:fixed|resolved)|hypothetical)\b/i.test(subject)) return [];
|
||||
const title = subject.replace(/[`*]/g, '');
|
||||
// A concise decision title can name the design choice while its own
|
||||
// current explanation states the defect. Keep these three forms bound to
|
||||
// that explanation and the same native option's concrete repair.
|
||||
const decisionTitle = title.replace(/^D[1-9]\d*\s*[—–:-]\s*/, '');
|
||||
const explainedSeed: Seed | undefined = /^Reduce the new-class count before we review the rest\?$/.test(decisionTitle) ? 'complexity'
|
||||
: /^Who is allowed to write to the auth cache\?$/.test(decisionTitle) ? 'shared-cache'
|
||||
: /^How should validateAndDispatch\(\) handle errors\?$/.test(decisionTitle) ? 'swallowed-errors' : undefined;
|
||||
if (explainedSeed) {
|
||||
const ordinal = /^D([1-9]\d*)\s*[—–:-]/.exec(title)?.[1];
|
||||
const status = '(?:withdrawn|rejected|cancelled|canceled|superseded|resolved|fixed|optional|hypothetical|unproven|not current|no longer current)';
|
||||
const owner = `(?:(?:this|the|that) (?:finding|issue|gap|defect|assessment|explanation)${ordinal ? `|D${ordinal}` : ''})`;
|
||||
const current = (value: string, option = false) => {
|
||||
const subject = option ? `(?:${owner}|(?:this|the|that) (?:option|action|remedy|correction))` : owner;
|
||||
const scalar = new RegExp(`((?:^|[.!?]\\s+|\\n)[\\t ]*(?:Correction:\\s*)?${subject} (?:is|was|has been) )["“'‘\x60](${status})["”'’\x60]`, 'gim');
|
||||
return prose(value.replace(scalar, '$1$2'), true).replace(/"[^"\n]*"|“[^”\n]*”|(?<![A-Za-z0-9])'[^'\n]*'(?![A-Za-z0-9])|‘[^’\n]*’/g, '');
|
||||
};
|
||||
const framed = /(?:^|[.!?;:]\s+|\n)(?:(?:Project\/branch\/task|ELI10):\s*)?(?:Source(?: excerpt| material)?|Quoted(?: source)?|Historical(?: assessment| example)?|If approved|Once approved|When approved|Pending approval|Assuming approval|Provided approval)[,:.]?\s/i;
|
||||
const active = (value: string, option = false) => {
|
||||
const text = current(value, option), subject = option ? `(?:${owner}|(?:this|the|that) (?:option|action|remedy|correction))` : owner;
|
||||
return !framed.test(text) && !new RegExp(`(?:^|[.!?;]\\s+|\\n)(?:Correction:\\s*)?${subject} (?:is|was|has been) ${status}\\b`, 'i').test(text) &&
|
||||
!(option && /(?:^|[.!?;]\s+|\n)(?:Correction:\s*)?(?:do not|don't|never|skip|cancel|withdraw) (?:drop|reduce|inject|flatten|rethrow|make|apply)\b/i.test(text));
|
||||
};
|
||||
const text = current(q.question), explanations = [...text.matchAll(/^ELI10: (.+)$/gm)];
|
||||
const prefix = text.slice(text.indexOf('\n') + 1, explanations[0]?.index ?? 0).trim().split('\n').filter(Boolean);
|
||||
if (explanations.length !== 1 || prefix.length !== 1 || !/^Project\/branch\/task: \S/.test(prefix[0]!) || !active(q.question)) return [];
|
||||
const explanation = explanations[0]![1]!;
|
||||
const options = q.options.filter(o => active(`${o.label}\n${o.description ?? ''}`, true))
|
||||
.map(o => ({ label: current(o.label).replace(/^[1-9]\d*[A-D]\s+/, ''), description: current(o.description ?? '') }));
|
||||
if (explainedSeed === 'complexity') {
|
||||
const wrapper = /^The plan invents a new cache wrapper \(([A-Za-z][\w]*)\) and a new token store on top of a cache adapter that already does tenant keying, expiry, and invalidation\./.exec(explanation);
|
||||
return wrapper && options.some(o => /^Reduce\b/.test(o.label) &&
|
||||
new RegExp(`\\bdrop ${wrapper[1]}\\/TokenStore classes\\.`).test(o.description)) ? [explainedSeed] : [];
|
||||
}
|
||||
if (explainedSeed === 'shared-cache') return /\bAuthBroker and SessionMint both mutating one backing cache\b/.test(prefix[0]!) &&
|
||||
/^Two services write to the same cache and nothing orders their writes\./.test(explanation) &&
|
||||
!/(?:^|[.!?]\s+|\n)(?:Correction:\s*)?(?:the|this) (?:cache|writes|writers) (?:is|are|have been) (?:now ordered|now serialized|no longer shared)\b/i.test(text) &&
|
||||
options.some(o => /^Single writer\b/.test(o.label) && /^SessionMint writes\b/.test(o.description) && /\bAuthBroker reads\b/.test(o.description)) ? [explainedSeed] : [];
|
||||
return /\bvalidateAndDispatch\(\) is [1-9]\d* lines, (?:three|[1-9]\d*) nested try\/catch blocks, each catch swallows a different error class\b/.test(prefix[0]!) &&
|
||||
/^Right now when something goes wrong inside validation, the code catches the problem and keeps going as if nothing happened\./.test(explanation) &&
|
||||
options.some(o => /^Flatten \+ typed errors\b/.test(o.label) && /\bLinear pipeline, typed error classes, one fail-closed boundary\b/.test(o.description) ||
|
||||
/^Rethrow, one outer catch$/.test(o.label) && /\brethrow typed errors; outer catch denies\b/.test(o.description)) ? [explainedSeed] : [];
|
||||
}
|
||||
const offered = q.options.map(o => `${o.label} ${o.description ?? ''}`).join('\n');
|
||||
const directAction = title.match(/\b(?:should|shall|can|do|would)\s+(?:we|I)\s+([^?]+)\?\s*$/i)?.[1];
|
||||
const action = (re: RegExp) => re.test(offered) || Boolean(directAction && new RegExp(`^(?:${re.source})`, re.flags).test(directAction));
|
||||
@@ -523,6 +565,116 @@ function declaredLegacyCharacterization(text: string): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
// A mandatory current-output baseline can land before all other tasks.
|
||||
// This obligation does not imply an identical suite on the flag-on path:
|
||||
// a plan may specify its rollout parity check separately.
|
||||
for (const section of current.filter(s => /^CRITICAL [—–-] regression \(mandatory, REGRESSION RULE\)$/i.test(s.title))) {
|
||||
const body = unquoted(section.body.join(' ')).replace(/\s+/g, ' ').trim();
|
||||
const rule = /^legacyAuthFlow\(\) is (?:live|existing|current) behavior being (?:changed|modified|refactored) with no covering test(?: \([^()!?]{1,120}\))?\. Before any (?:rewrite|refactor|change): ([A-Za-z][\w/.-]*\.test\.[jt]s) (?:records|captures|pins) (?:current|existing) outputs \(including quirks\) for [^.!?]{1,300} inputs\. This suite runs against the legacy path now\b/.exec(body);
|
||||
if (!rule || suiteWithdrawn || !snapshotSource.includes(rule[0]) || withdrawn(body) ||
|
||||
extractionSourceOwner(body) || approvalPending(body) || /\b(?:if|unless|maybe|might|could|optional|hypothetical|unproven)\b/i.test(body)) continue;
|
||||
for (const tasks of current.filter(s => s.title === 'Implementation Tasks')) {
|
||||
const taskBody = tasks.body.join('\n').trim(), blocks = taskBody.split(/\n(?=-\s)/);
|
||||
const prefix = blocks[0]?.trim() ?? '';
|
||||
if (!taskBody.startsWith('- ') && (!/^Synthesized from (?:this|the) review's findings\./.test(prefix) ||
|
||||
extractionSourceOwner(prefix) || approvalPending(prefix) || /\b(?:if|unless|assuming|provided|optional|hypothetical|unproven)\b/i.test(unquoted(prefix)))) continue;
|
||||
const ids = blocks.map(block => /^- (?:\[[ xX]\] )?(T[1-9]\d*)\b/.exec(block)?.[1]).filter(Boolean);
|
||||
if (ids.length !== new Set(ids).size) continue;
|
||||
for (const task of blocks) {
|
||||
const match = /^- (?:\[[ xX]\] )?(T[1-9]\d*)(?: \([^\n)]*\))? [—–-] [A-Za-z][\w/-]* [—–-] CRITICAL regression: characterization suite for legacyAuthFlow\(\) (?:current|existing|prior) behavior[\t ]*(?:\n|$)/.exec(task);
|
||||
const files = [...task.matchAll(/^\s+- Files: ([^\n]+)$/gm)];
|
||||
const verifies = [...task.matchAll(/^\s+- Verify: ([^\n]+)$/gm)];
|
||||
const verify = verifies[0];
|
||||
const preceding = unquoted(taskBody.slice(0, taskBody.indexOf(task))).trim().split('\n').at(-1) ?? '';
|
||||
if (!match || files.length !== 1 || files[0]![1] !== rule[1] || verifies.length !== 1 ||
|
||||
!/^suite green against unmodified legacy before any other task merges[\t ]*$/.test(verify![1]!) ||
|
||||
!snapshotSource.includes(task.replace(/\s+/g, ' ').trim()) ||
|
||||
extractionSourceOwner(preceding) || conditionalOwner(preceding) || approvalPending(preceding) ||
|
||||
extractionSourceOwner(task.slice(0, verify!.index)) || conditionalOwner(task.slice(0, verify!.index)) || approvalPending(task.slice(0, verify!.index)) ||
|
||||
/(?:^|\n)\s*(?:assuming|provided)\b/i.test(unquoted(task.slice(0, verify!.index)))) continue;
|
||||
const id = match[1]!;
|
||||
const status = '(?:withdrawn|rejected|cancelled|canceled|superseded|optional|not current|no longer current|not required|no longer required)';
|
||||
// Current scalar statuses retain their owner; whole-sentence quoted
|
||||
// history and conditional future statuses cannot cancel this baseline.
|
||||
const assessment = (value: string) => unquoted(value.replace(new RegExp(
|
||||
`((?:^|[.!?]\\s+|\\n)[\\t ]*(?:Correction:\\s*)?(?:${id}(?: (?:verification|baseline verification))?|(?:this|the) (?:(?:legacy|baseline|unchanged-code) )?(?:(?:regression|characterization) )?(?:suite|requirement|verification)) (?:is|was|has been) )["“'‘](${status})["”'’]`, 'gim'), '$1$2'))
|
||||
.split(/\n|[.!?]\s+/).filter(line => !conditionalOwner(line) && !approvalPending(line) && !/^\s*(?:assuming|provided)\b/i.test(line)).join('\n');
|
||||
const inactive = (value: string) => {
|
||||
const body = assessment(value).replace(/\b(?:this|the)\s+(?:(?:unchanged-code|baseline)\s+)?verification\b/gi, 'this requirement');
|
||||
return withdrawn(body, id) || new RegExp(`\\b(?:this|the) (?:suite|requirement) (?:is|was|has been) ${status}\\b`, 'i').test(body);
|
||||
};
|
||||
const cancelled = new RegExp(`\\b${id}(?: (?:verification|baseline verification))? (?:is|was|has been) ${status}\\b`, 'i');
|
||||
const statusRow = new RegExp(`^\\s*\\|\\s*${id}\\s*\\|\\s*["“'‘]?${status}["”'’]?\\s*\\|`, 'im');
|
||||
const changedFirst = new RegExp(`^(?:Correction:\\s*)?legacyAuthFlow\\(\\) (?:is|was|has been|will be) (?:modified|changed|rewritten|refactored) before ${id}\\b`, 'im');
|
||||
if (inactive(body) || inactive(task) || current.some(s => {
|
||||
const value = assessment(s.body.join('\n'));
|
||||
return statusRow.test(s.body.join('\n')) || cancelled.test(value) || changedFirst.test(value) ||
|
||||
new RegExp(`\\b(?:the|this) legacy (?:regression|characterization) (?:suite|tests?|requirement) (?:is|was|has been) ${status}\\b`, 'i').test(value);
|
||||
})) continue;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A directory-owned characterization task can name changed worktree steps
|
||||
// in its baseline check, with the baseline's own lane merging first.
|
||||
for (const section of current.filter(s => /^Tests(?: \([^\n)]+\))?$/.test(s.title))) {
|
||||
const body = unquoted(section.body.join(' ')).replace(/\s+/g, ' ').trim();
|
||||
const rule = /^CRITICAL regression suite \(mandatory, IRON RULE\)\. legacyAuthFlow\(\) is existing behavior being rewritten and the original plan had no regression coverage\. Before any rewrite, write a characterization suite that pins current behavior: [^.!?]{1,300}\. The suite runs against both the legacy path and the new flow \(via the flag\) for the whole rollout window\./.exec(body);
|
||||
if (!rule || suiteWithdrawn || !snapshotSource.includes(rule[0]) || withdrawn(body) ||
|
||||
/\b(?:if|unless|maybe|might|could|optional|hypothetical|unproven)\b/i.test(rule[0])) continue;
|
||||
for (const tasks of current.filter(s => s.title === 'Implementation Tasks')) {
|
||||
const taskBody = tasks.body.join('\n').trim(), blocks = taskBody.split(/\n(?=-\s)/), prefix = blocks[0]?.trim() ?? '';
|
||||
if (!taskBody.startsWith('- ') && (!/^Synthesized from (?:this|the) review's findings\./.test(prefix) || extractionSourceOwner(prefix) ||
|
||||
approvalPending(prefix) || /\b(?:if|unless|assuming|provided|optional|hypothetical|unproven)\b/i.test(unquoted(prefix)))) continue;
|
||||
const ids = blocks.map(block => /^- (?:\[[ xX]\] )?(T[1-9]\d*)\b/.exec(block)?.[1]).filter(Boolean);
|
||||
if (ids.length !== new Set(ids).size) continue;
|
||||
for (const task of blocks) {
|
||||
const match = /^- (?:\[[ xX]\] )?(T[1-9]\d*)(?: \([^\n)]*\))? [—–-] ([A-Za-z][\w/-]*) [—–-] Write the CRITICAL characterization suite for legacyAuthFlow\(\) before any rewrite; run it against legacy and new flow[\t ]*(?:\n|$)/.exec(task);
|
||||
const files = [...task.matchAll(/^\s+- Files: ([^\n]+)$/gm)];
|
||||
const verifies = [...task.matchAll(/^\s+- Verify: suite green on legacy path before (S[1-9]\d*)\/(S[1-9]\d*) land; green on both paths after[\t ]*$/gm)];
|
||||
const verify = verifies[0], preceding = unquoted(taskBody.slice(0, taskBody.indexOf(task))).trim().split('\n').at(-1) ?? '';
|
||||
if (!match || files.length !== 1 || files[0]![1] !== `${match[2]}/, router flag stub` || verifies.length !== 1 ||
|
||||
(task.match(/^\s+- Verify:/gm)?.length ?? 0) !== 1 || !snapshotSource.includes(task.replace(/\s+/g, ' ').trim()) ||
|
||||
extractionSourceOwner(preceding) || conditionalOwner(preceding) || approvalPending(preceding) ||
|
||||
extractionSourceOwner(task.slice(0, verify!.index)) || conditionalOwner(task.slice(0, verify!.index)) || approvalPending(task.slice(0, verify!.index)) ||
|
||||
/(?:^|\n)\s*(?:assuming|provided)\b/i.test(unquoted(task.slice(0, verify!.index)))) continue;
|
||||
for (const strategy of current.filter(s => s.title === 'Worktree parallelization strategy')) {
|
||||
const schedule = unquoted(strategy.body.join('\n'));
|
||||
const steps = [...schedule.matchAll(/^\| (S[1-9]\d*)\b/gm)].map(row => row[1]);
|
||||
if (steps.length !== new Set(steps).size) continue;
|
||||
const baseline = /^\| (S[1-9]\d*) Regression suite for legacyAuthFlow\(\) \| ([A-Za-z][\w/-]*) \| [—–-] \|$/m.exec(schedule);
|
||||
const lanes = baseline ? [...schedule.matchAll(new RegExp(`^\\s*Lane ([A-Z]): ${baseline[1]} \\(independent\\)$`, 'gm'))] : [];
|
||||
const lane = lanes.length === 1 ? lanes[0] : undefined;
|
||||
const order = lane && new RegExp(`^Execution order: launch [A-Z](?:, [A-Z])+ in parallel worktrees\\. Merge ${lane[1]} first \\(it is\\s+pure tests and gates the rewrite\\)\\.`, 'm').exec(schedule);
|
||||
if (!baseline || baseline[2] !== match[2] || !lane || !order || verify![1] === verify![2] ||
|
||||
!new RegExp(`^\\| ${verify![1]} AuthBroker \\+ SessionMint \\| [^|]+ \\| [^|]+ \\|$`, 'm').test(schedule) ||
|
||||
!new RegExp(`^\\| ${verify![2]} Flattened dispatcher \\+ flag router \\+ fallback \\| [^|]+ \\| [^|]+ \\|$`, 'm').test(schedule) ||
|
||||
extractionSourceOwner(schedule.slice(0, order.index)) || conditionalOwner(schedule.slice(0, order.index)) || approvalPending(schedule.slice(0, order.index)) ||
|
||||
!snapshotSource.includes(order[0].replace(/\s+/g, ' '))) continue;
|
||||
const id = `(?:${match[1]}|${baseline[1]})`, status = '(?:withdrawn|rejected|cancelled|canceled|superseded|optional|not current|no longer current|not required|no longer required)';
|
||||
const assessment = (value: string) => unquoted(value.replace(new RegExp(
|
||||
`((?:^|[.!?]\\s+|\\n)[\\t ]*(?:Correction:\\s*)?(?:${id}(?: (?:verification|baseline verification))?|(?:this|the) (?:(?:legacy|baseline|unchanged-code) )?(?:(?:regression|characterization) )?(?:suite|requirement|verification)) (?:is|was|has been) )["“'‘](${status})["”'’]`, 'gim'), '$1$2'))
|
||||
.replace(/(?<![A-Za-z0-9])'[^'\n]*'(?![A-Za-z0-9])|‘[^’\n]*’/g, '')
|
||||
.split(/\n|[.!?]\s+/).filter(line => !conditionalOwner(line) && !approvalPending(line) && !/^\s*(?:assuming|provided)\b/i.test(line)).join('\n');
|
||||
const inactive = (value: string) => {
|
||||
const text = assessment(value).replace(/\b(?:this|the)\s+(?:(?:unchanged-code|baseline)\s+)?verification\b/gi, 'this requirement');
|
||||
return withdrawn(text, id) || new RegExp(`\\b(?:this|the) (?:suite|requirement) (?:is|was|has been) ${status}\\b`, 'i').test(text);
|
||||
};
|
||||
const cancelled = new RegExp(`\\b${id}(?: (?:verification|baseline verification))? (?:is|was|has been) ${status}\\b`, 'i');
|
||||
const statusRow = new RegExp(`^\\s*\\|\\s*${id}\\s*\\|\\s*["“'‘]?${status}["”'’]?\\s*\\|`, 'im');
|
||||
const changedFirst = new RegExp(`^(?:Correction:\\s*)?legacyAuthFlow\\(\\) (?:is|was|has been|will be) (?:modified|changed|rewritten|refactored) before ${id}\\b`, 'im');
|
||||
if (inactive(body) || inactive(task) || current.some(s => {
|
||||
const value = assessment(s.body.join('\n'));
|
||||
return statusRow.test(s.body.join('\n')) || cancelled.test(value) || changedFirst.test(value) ||
|
||||
new RegExp(`\\b(?:the|this) (?:legacy (?:regression|characterization) (?:suite|tests?|requirement)|(?:baseline|unchanged-code) verification) (?:is|was|has been) ${status}\\b`, 'i').test(value);
|
||||
})) continue;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -218,6 +218,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'test/fixtures/autoplan-final-gate-ao.json',
|
||||
"test/autoplan-clipped-suffix-aq.test.ts", "test/fixtures/autoplan-clipped-suffix-aq.json", "test/design-scope-entry-aq.test.ts",
|
||||
"test/helpers/autoplan-preconfigured-fixture.ts", "test/autoplan-preconfigured-onboarding-ar.test.ts",
|
||||
"test/autoplan-artifact-stall-as.test.ts", "test/fixtures/autoplan-artifact-stall-as.json",
|
||||
],
|
||||
|
||||
// Per-finding AskUserQuestion count + review-report-at-bottom assertion.
|
||||
@@ -267,6 +268,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
"test/fixtures/eng-golden-parity-an.json",
|
||||
"test/eng-injected-export-aq.test.ts", "test/fixtures/eng-injected-export-aq.json", "test/eng-staged-regression-aq.test.ts", "test/fixtures/eng-staged-regression-aq.md", "test/eng-library-hooks-aq.test.ts", "test/fixtures/eng-library-hooks-aq.json",
|
||||
"test/eng-before-rewrite-ar.test.ts", "test/fixtures/eng-before-rewrite-ar.md",
|
||||
"test/eng-declarative-as.test.ts", "test/fixtures/eng-declarative-as.json", "test/eng-mandatory-baseline-as.test.ts", "test/fixtures/eng-mandatory-baseline-as.md", "test/helpers/eng-cache-writer-decision.ts", "test/eng-cache-writes-as.test.ts", "test/fixtures/eng-cache-writes-as.json", "test/eng-retry-coverage-as.test.ts", "test/fixtures/eng-retry-coverage-as.json", "test/fixtures/eng-retry-baseline-as.md",
|
||||
],
|
||||
'plan-design-finding-count': ['test/pty-screen-unicode-ap.test.ts', 'test/design-crop-gutter-ap.test.ts', 'test/fixtures/design-crop-gutter-ap.json', 'test/helpers/dx-selected-navigation.ts', 'test/dx-selected-navigation-ap.test.ts', 'test/fixtures/dx-selected-navigation-ap.json', 'test/dx-manual-handoff-ao.test.ts', 'test/fixtures/dx-manual-handoff-ao.json', 'bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/design-count-review.ts', 'test/design-count-review.test.ts', 'test/fixtures/design-review-j-calls.json', 'test/helpers/design-count-outside.ts', 'test/design-count-outside.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts', 'test/fixtures/design-review-l-calls.json', 'test/design-completion-handoff.test.ts', 'test/fixtures/design-handoff-l-calls.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/design-review-n-calls.json', 'test/design-completion-handoff-scored.test.ts', 'test/fixtures/design-handoff-n-calls.json', 'test/fixtures/design-handoff-q-calls.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/design-completion-handoff-u.test.ts', 'test/fixtures/design-handoff-u-calls.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/helpers/design-artifact-question.ts', 'test/design-artifact-question.test.ts', 'test/fixtures/design-artifacts-w-calls.json', 'test/fixtures/design-outside-y-calls.json', 'test/fixtures/design-boundaries-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/design-gap-z-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/fixtures/plan-count-permission-ae.json', 'test/fixtures/plan-count-permission-target-ad-v2.json', 'test/design-count-ad-v2.test.ts', 'test/fixtures/design-count-ad-v2.json', 'test/design-first-decision-af.test.ts', 'test/fixtures/design-first-decision-af.json', 'test/fixtures/design-first-decision-af-retry.json', 'test/fixtures/plan-count-permission-ah.json', 'test/design-first-issue-ai.test.ts', 'test/fixtures/design-first-issue-ai.json', 'test/design-primary-action-aj.test.ts', 'test/fixtures/design-primary-action-aj.json', 'test/fixtures/design-future-todo-aj.json',
|
||||
'test/design-primary-contract-ak.test.ts',
|
||||
@@ -287,6 +289,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
"test/design-primary-assignment-ao.test.ts",
|
||||
"test/fixtures/design-primary-assignment-ao.json",
|
||||
"test/design-primary-header-aq.test.ts", "test/fixtures/design-primary-header-aq.json", "test/design-scope-entry-aq.test.ts",
|
||||
"test/design-primary-group-as.test.ts", "test/fixtures/design-primary-group-as-calls.json",
|
||||
],
|
||||
'plan-devex-finding-count': ['test/pty-screen-unicode-ap.test.ts', 'test/design-crop-gutter-ap.test.ts', 'test/fixtures/design-crop-gutter-ap.json', 'test/helpers/dx-selected-navigation.ts', 'test/dx-selected-navigation-ap.test.ts', 'test/fixtures/dx-selected-navigation-ap.json', 'test/dx-manual-handoff-ao.test.ts', 'test/fixtures/dx-manual-handoff-ao.json', 'bin/gstack-config', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/plan-count-completion.test.ts', 'test/plan-count-dx-handoff.test.ts', 'test/fixtures/devex-handoff-n-call.json', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/helpers/devex-count-fixture.ts', 'test/devex-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts', 'test/fixtures/dx-prerequisite-r-call.json', 'test/fixtures/devex-review-l-calls.json', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/fixtures/devex-review-n-calls.json', 'test/plan-count-empty-review.test.ts', 'test/fixtures/ceo-count-s-distinct.json', 'test/fixtures/plan-count-design-questionless-report.md', 'test/devex-output-o.test.ts', 'test/fixtures/devex-review-o-calls.json', 'test/fixtures/devex-output-o-retry-call.json', 'test/devex-setup-remedy-o.test.ts', 'test/fixtures/devex-review-o-retry-calls.json', 'test/plan-count-dx-handoff-o.test.ts', 'test/fixtures/devex-handoff-o-call.json', 'test/eng-devex-s-count.test.ts', 'test/fixtures/eng-devex-s-first-calls.json', 'test/fixtures/eng-devex-s-retry-calls.json', 'test/fixtures/devex-review-t-calls.json', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-preview-footer.test.ts', 'test/fixtures/ceo-preview-u-call.json', 'test/fixtures/ceo-preview-u-screen.txt', 'test/fixtures/devex-count-u-calls.json', 'test/fixtures/devex-count-u-retry-calls.json', 'test/fixtures/devex-empathy-v-calls.json', 'test/fixtures/devex-handoff-v-call.json', 'test/fixtures/design-preview-v-screen.txt', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/fixtures/devex-count-y-calls.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/fixtures/devex-count-z-calls.json', 'test/fixtures/devex-handoff-z-call.json', 'test/review-handoffs-aa.test.ts', 'test/fixtures/review-handoff-aa-ceo.json', 'test/fixtures/review-handoff-aa-dx.json', 'test/devex-empathy-ab.test.ts', 'test/fixtures/devex-empathy-ab-calls.json', 'test/devex-ac-accounting.test.ts', 'test/fixtures/devex-ac-first-attempt-calls.json', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/fixtures/plan-count-permission-ae.json', 'test/fixtures/plan-count-permission-target-ad-v2.json', 'test/devex-reconfirmation-ad-v2.test.ts', 'test/fixtures/devex-reconfirmation-ad-v2.json', 'test/plan-count-history.test.ts', 'test/helpers/devex-seed-coverage.ts', 'test/devex-seed-coverage.test.ts', 'test/fixtures/devex-seed-coverage-ad-v3.json', 'test/fixtures/plan-count-permission-ah.json',
|
||||
'test/dx-signature-identity-ak.test.ts',
|
||||
@@ -297,6 +300,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'test/fixtures/plan-count-quoted-frame-ak.json',
|
||||
"test/fixtures/dx-declarative-choices-am.json",
|
||||
"test/dx-declarative-stage-ar.test.ts", "test/fixtures/dx-declarative-stage-ar.json",
|
||||
"test/dx-asserted-defect-as.test.ts", "test/fixtures/dx-asserted-defect-as.json", "test/fixtures/dx-asserted-defect-as-retry.json",
|
||||
],
|
||||
|
||||
// Gate-tier reviewCount-floor counterparts. Catch the May 2026 transcript
|
||||
@@ -327,6 +331,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'test/plan-count-quoted-frame-ak.test.ts',
|
||||
'test/fixtures/plan-count-quoted-frame-ak.json',
|
||||
"test/eng-injected-export-aq.test.ts", "test/fixtures/eng-injected-export-aq.json", "test/eng-library-hooks-aq.test.ts", "test/fixtures/eng-library-hooks-aq.json",
|
||||
"test/eng-declarative-as.test.ts", "test/fixtures/eng-declarative-as.json", "test/helpers/eng-cache-writer-decision.ts", "test/eng-cache-writes-as.test.ts", "test/fixtures/eng-cache-writes-as.json",
|
||||
],
|
||||
'plan-ceo-split-overflow': ['test/pty-screen-unicode-ap.test.ts', 'test/design-crop-gutter-ap.test.ts', 'test/fixtures/design-crop-gutter-ap.json', 'bin/gstack-config', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-native-input.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-transcript.ts', 'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/plan-count-transcript.test.ts', 'test/helpers/plan-count-artifacts.ts', 'test/plan-count-artifacts.test.ts', 'test/helpers/eval-store.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/plan-count-timeout.test.ts', 'test/plan-count-navigation-r.test.ts', 'test/plan-count-prerequisite-n.test.ts', 'test/fixtures/ceo-prerequisite-n-call.json', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts', 'test/plan-count-checkbox.test.ts', 'test/fixtures/ceo-checkbox-l.screen.txt', 'test/helpers/plan-count-file-permission.ts', 'test/plan-count-file-permission.test.ts', 'test/fixtures/plan-count-edit-permission-t.json', 'test/plan-count-owned-permission.test.ts', 'test/fixtures/plan-count-owned-permission-v.json', 'test/fixtures/ceo-questionless-w-native.json', 'test/plan-count-truncated-question.test.ts', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/plan-count-permission-ac.test.ts', 'test/fixtures/plan-count-permission-ac.json', 'test/fixtures/plan-count-permission-ad.json', 'test/fixtures/plan-count-permission-ae.json', 'test/fixtures/plan-count-permission-target-ad-v2.json', 'test/fixtures/plan-count-permission-ah.json',
|
||||
'test/plan-count-crop-ak.test.ts',
|
||||
@@ -480,9 +485,11 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
],
|
||||
'review-coverage-audit': ['review/**', 'test/fixtures/coverage-audit-fixture.ts', 'test/skill-e2e-coverage-audit.test.ts', 'test/helpers/coverage-audit-evidence.ts', 'test/coverage-audit-evidence.test.ts', 'test/fixtures/coverage-audit-ae.json', 'test/coverage-audit-af.test.ts', 'test/fixtures/coverage-audit-af.json',
|
||||
"test/coverage-shell-display-aq.test.ts", "test/fixtures/coverage-shell-display-aq.json",
|
||||
"test/coverage-diagram-legend-as.test.ts", "test/fixtures/coverage-diagram-legend-as.json",
|
||||
],
|
||||
'plan-eng-coverage-audit': ['test/eng-scope-entry-ap.test.ts', 'plan-eng-review/**', 'test/fixtures/coverage-audit-fixture.ts', 'test/skill-e2e-coverage-audit.test.ts', 'test/helpers/coverage-audit-evidence.ts', 'test/coverage-audit-evidence.test.ts', 'test/fixtures/coverage-audit-ae.json', 'test/coverage-audit-af.test.ts', 'test/fixtures/coverage-audit-af.json',
|
||||
"test/coverage-shell-display-aq.test.ts", "test/fixtures/coverage-shell-display-aq.json",
|
||||
"test/coverage-diagram-legend-as.test.ts", "test/fixtures/coverage-diagram-legend-as.json",
|
||||
],
|
||||
'ship-triage': ['ship/**', 'bin/gstack-repo-mode', 'test/skill-e2e-triage.test.ts'],
|
||||
'ship-docsync': ['ship/**', 'document-release/**', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/sections.ts', 'test/skill-e2e-ship-docsync.test.ts'],
|
||||
|
||||
@@ -166,6 +166,7 @@ describeE2E('/autoplan native chain ordering (periodic)', () => {
|
||||
commandStartedAt, transcriptStatus: transcript.status, publicTools,
|
||||
}, seenArtifactPermissions) ?? publishedAutoplanArtifactPermissionInput(visible, {
|
||||
cwd: tempDir, ownedStateRoot: session.hermeticSkillStateRoot, commandStartedAt,
|
||||
ownedNativePlansRoot: session.hermeticConfigDir ? path.join(session.hermeticConfigDir, 'plans') : undefined,
|
||||
transcriptStatus: transcript.status, publicTools, pending: pendingArtifact, viewportCapturedAt,
|
||||
}, seenArtifactPermissions) ?? pendingAutoplanArtifactPermissionInput(visible, {
|
||||
cwd: tempDir, ownedStateRoot: session.hermeticSkillStateRoot, commandStartedAt,
|
||||
|
||||
Reference in new issue
Block a user