From a6eaafacb846bdb018a1ead13985be9c05e2c2df Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Thu, 10 Sep 2026 05:34:46 +0000 Subject: [PATCH] test: repair seeded workflow delivery and completion evidence --- test/auto-decide-saved-ai.test.ts | 80 +++ test/autoplan-edit-prefix-ai.test.ts | 102 ++++ test/ceo-section-choice-ai.test.ts | 104 ++++ test/design-first-issue-ai.test.ts | 132 +++++ test/eng-declared-regression-ai.test.ts | 197 ++++++++ test/eng-seeded-completion-ai.test.ts | 177 +++++++ test/fixtures/auto-decide-retry-ai.json | 454 ++++++++++++++++++ test/fixtures/auto-decide-saved-ai.json | 291 +++++++++++ test/fixtures/autoplan-edit-prefix-ai.json | 53 ++ test/fixtures/ceo-section-choice-ai.json | 264 ++++++++++ test/fixtures/design-first-issue-ai.json | 441 +++++++++++++++++ test/fixtures/eng-declared-regression-ai.json | 315 ++++++++++++ test/fixtures/eng-seeded-completion-ai.json | 20 + test/fixtures/outside-background-ai.json | 201 ++++++++ test/fixtures/sdk-original-order-ai.json | 19 + test/helpers/autoplan-artifact-permission.ts | 24 + test/helpers/ceo-section-loading-fixture.ts | 182 +++++++ test/helpers/claude-pty-runner.ts | 160 ++++-- test/helpers/design-count-review.ts | 47 +- test/helpers/eng-seeded-coverage.ts | 73 +++ test/helpers/native-auto-decide.ts | 25 +- test/helpers/outside-voice-evidence.ts | 17 +- test/helpers/plan-floor-target.ts | 65 +++ test/helpers/touchfiles-data.ts | 42 +- test/outside-background-ai.test.ts | 103 ++++ test/plan-count-prerequisite-n.test.ts | 20 +- test/plan-floor-target.test.ts | 145 ++++++ test/sdk-original-order-ai.test.ts | 143 ++++++ 28 files changed, 3824 insertions(+), 72 deletions(-) create mode 100644 test/auto-decide-saved-ai.test.ts create mode 100644 test/autoplan-edit-prefix-ai.test.ts create mode 100644 test/ceo-section-choice-ai.test.ts create mode 100644 test/design-first-issue-ai.test.ts create mode 100644 test/eng-declared-regression-ai.test.ts create mode 100644 test/eng-seeded-completion-ai.test.ts create mode 100644 test/fixtures/auto-decide-retry-ai.json create mode 100644 test/fixtures/auto-decide-saved-ai.json create mode 100644 test/fixtures/autoplan-edit-prefix-ai.json create mode 100644 test/fixtures/ceo-section-choice-ai.json create mode 100644 test/fixtures/design-first-issue-ai.json create mode 100644 test/fixtures/eng-declared-regression-ai.json create mode 100644 test/fixtures/eng-seeded-completion-ai.json create mode 100644 test/fixtures/outside-background-ai.json create mode 100644 test/fixtures/sdk-original-order-ai.json create mode 100644 test/helpers/plan-floor-target.ts create mode 100644 test/outside-background-ai.test.ts create mode 100644 test/plan-floor-target.test.ts create mode 100644 test/sdk-original-order-ai.test.ts diff --git a/test/auto-decide-saved-ai.test.ts b/test/auto-decide-saved-ai.test.ts new file mode 100644 index 000000000..5906ed268 --- /dev/null +++ b/test/auto-decide-saved-ai.test.ts @@ -0,0 +1,80 @@ +import { expect, test } from 'bun:test'; +import { findNativeAutoDecision } from './helpers/native-auto-decide'; +import captured from './fixtures/auto-decide-saved-ai.json'; +const clone=()=>structuredClone(captured) as any; +const decision=(f=clone())=>findNativeAutoDecision(f.transcript,f.tools,f.options); +const message=(f:any)=>f.transcript.assistantMessages.find((m:any)=>m.text.includes('Auto-decided')); + +test('actual saved mode preference annotation is a completed native auto-decision',()=>{ + const f=clone(), result=decision(f); + expect(result).not.toBeNull(); + expect(result!.option).toBe('HOLD SCOPE'); + expect(message(f).text).toContain(result!.annotation); + expect(f.transcript.calls).toEqual([]); +}); + +test('saved preference is bound to this invoked skill and an agreeing current mode',()=>{ + for(const change of [ + (s:string)=>s.replace('`plan-ceo-review-mode`','`plan-design-review-mode`'), + (s:string)=>s.replace('`plan-ceo-review-mode`','`plan-ceo-review-routing`'), + (s:string)=>s.replace('**Review mode: HOLD SCOPE.**','**Review mode: SCOPE EXPANSION.**'), + (s:string)=>s.replace('**Review mode: HOLD SCOPE.**\n\n',''), + (s:string)=>s.replace('"Select review mode"','"Select report folder"'), + (s:string)=>s.replace('(your saved preference on','(a proposed preference on'), + (s:string)=>s.replace('Change with /plan-tune.',''), + (s:string)=>s.replace('Auto-decided','I will auto-decide'), + ]) {const f=clone();message(f).text=change(message(f).text);expect(decision(f)).toBeNull();} +}); + +test('prefixed examples, quotations and hypothetical notices do not assert a current choice',()=>{ + for(const change of [ + (s:string)=>'Example:\n\n'+s, + (s:string)=>'```text\n'+s+'\n```', + (s:string)=>s.split('\n').map(l=>'> '+l).join('\n'), + (s:string)=>s.replace('Heads-up from the preamble: unshipped work on this branch','Heads-up from the preamble: a hypothetical example'), + (s:string)=>s.replace('Auto-decided "Select',' Auto-decided "Select'), + (s:string)=>s.replace('Auto-decided "Select','If approved, Auto-decided "Select'), + ]) {const f=clone();message(f).text=change(message(f).text);expect(decision(f)).toBeNull();} +}); + +test('failed loads, foreign sessions, actual questions and later withdrawals retain precedence',()=>{ + for(const mutate of [ + (f:any)=>{f.options.sessionId='foreign';}, + (f:any)=>{const use=f.tools.find((t:any)=>t.kind==='use'&&t.name==='Skill');f.tools.find((t:any)=>t.kind==='result'&&t.toolUseId===use.toolUseId).isError=true;}, + (f:any)=>{f.transcript.calls.push({sessionId:f.options.sessionId,toolUseId:'actual-question'});}, + (f:any)=>{f.options.now=Date.parse(message(f).timestamp)-1;}, + (f:any)=>{message(f).text+='\n\nCorrection: I withdraw this decision.';}, + (f:any)=>{message(f).text+='\n\n**Review mode: SCOPE EXPANSION.**';}, + ]) {const f=clone();mutate(f);expect(decision(f)).toBeNull();} +}); + +import { E2E_TOUCHFILES } from './helpers/touchfiles-data'; +test('new evidence inputs retain every native observation caller',()=>{ + const owners=['plan-ceo-review-plan-mode','plan-eng-review-plan-mode','plan-design-review-plan-mode','plan-devex-review-plan-mode','plan-mode-no-op','auto-decide-preserved','conductor-prose']; + for(const file of ['test/auto-decide-saved-ai.test.ts','test/fixtures/auto-decide-saved-ai.json','test/fixtures/auto-decide-retry-ai.json']) + expect(Object.entries(E2E_TOUCHFILES).filter(([,paths])=>paths.includes(file)).map(([name])=>name)).toEqual(owners); +}); + +import retry from './fixtures/auto-decide-retry-ai.json'; +test('actual retry mode-decision heading retains its own annotation, excluding prior foreign text',()=>{ + const f:any=structuredClone(retry), actual=decision(f); + expect(actual).not.toBeNull();expect(actual!.sessionId).toBe(f.options.sessionId); + expect(actual!.option).toBe('HOLD SCOPE'); + expect(actual!.annotation).toContain('(your preference)'); + const own=f.transcript.assistantMessages.filter((m:any)=>m.sessionId===f.options.sessionId); + f.transcript.assistantMessages=f.transcript.assistantMessages.filter((m:any)=>m.sessionId!==f.options.sessionId); + expect(decision(f)).toBeNull();expect(own.length).toBeGreaterThan(0); +}); + +test('retry heading cannot supply a hypothetical, different decision, or withdrawn selection',()=>{ + for(const change of [ + (s:string)=>'Example:\n\n'+s, + (s:string)=>s.replace('Review mode for the deterministic','Review mode for the hypothetical'), + (s:string)=>s.replace('D1 — Review mode','D1 — Report destination'), + (s:string)=>s.replace('Auto-decided "Review mode:', 'Auto-decided "Report destination:'), + (s:string)=>s.replace('→ **HOLD SCOPE**','→ **Save a file**'), + (s:string)=>s+'\n\nCorrection: I withdraw this selection.', + (s:string)=>s+'\n\n**Review mode: SCOPE EXPANSION.**', + (s:string)=>s.replace('Heads-up from gstack: there is unshipped work on this branch','Heads-up from gstack: here is an example'), + ]) {const f:any=structuredClone(retry),m=f.transcript.assistantMessages.find((m:any)=>m.sessionId===f.options.sessionId&&m.text.includes('Auto-decided'));m.text=change(m.text);expect(decision(f)).toBeNull();} +}); diff --git a/test/autoplan-edit-prefix-ai.test.ts b/test/autoplan-edit-prefix-ai.test.ts new file mode 100644 index 000000000..73aeacc63 --- /dev/null +++ b/test/autoplan-edit-prefix-ai.test.ts @@ -0,0 +1,102 @@ +import { afterEach, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import captured from './fixtures/autoplan-edit-prefix-ai.json'; +import { autoplanArtifactPermissionInput, pendingAutoplanArtifactPermissionInput, autoplanArtifactMenuKey } from './helpers/autoplan-artifact-permission'; +import type { NativePublicToolEvent } from './helpers/plan-count-transcript'; +import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles'; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); }); +function replay() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'autoplan-edit-prefix-')); roots.push(root); + const cwd = path.join(root, path.basename(captured.cwd)), ownedStateRoot = path.join(root, 'home', '.gstack'); + const events = structuredClone(captured.events) as NativePublicToolEvent[]; + const latest = events.find(e => e.kind === 'use' && e.toolUseId === captured.pending.toolUseId)! + const original = latest.input!.file_path as string, file = original.replace(captured.ownedStateRoot, ownedStateRoot); + fs.mkdirSync(cwd, { recursive: true }); fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, captured.before); + const time = new Date(Date.parse(latest.timestamp) - 1000); fs.utimesSync(file, time, time); + for (const event of events) if (event.input?.file_path === original) event.input.file_path = file; + const context = { cwd, ownedStateRoot, commandStartedAt: Date.parse(events[0]!.timestamp) - 1, + now: Date.parse(captured.viewportCapturedAt), viewportCapturedAt: Date.parse(captured.viewportCapturedAt), transcriptStatus: 'ready', publicTools: events }; + const viewport = captured.viewport.replace(/^ …[^\n]+$/m, ' …' + path.relative(ownedStateRoot, file)); + const header = viewport.lastIndexOf('\n● Update(') + 1; + return { root, file, current: latest, context, viewport, prefix: viewport.slice(0, header), panel: viewport.slice(header) }; +} +const pick = (r: ReturnType, seen = new Set()) => autoplanArtifactPermissionInput(r.viewport, r.context, seen); + +test('the exact retained prior diff output does not hide the current published owned edit', () => { + const r = replay(); + expect(r.prefix.split('\n')).toHaveLength(17); + expect(pick(r)).toEqual({ input: '1\r', signature: r.current.sessionId + ':' + r.current.toolUseId, file: r.file }); + r.viewport = r.panel; + expect(pick(r)?.input).toBe('1\r'); +}); + +test('completed diff rows are ignored only before one complete current native panel', () => { + for (const prefix of [' 1 +Previous completed output\n\n', ' +cropped prior row\n 2 +next prior row\n +wrapped row\n\n', ' 1 -Old value\n 1 +New value\n\n']) { + const r = replay(); r.viewport = prefix + r.panel; expect(pick(r)?.input).toBe('1\r'); + } +}); + +test('competing headers, previous panels, misleading prose and quotes remain rejected', () => { + for (const prefix of [ + '● Update(/tmp/foreign.md)\n\n', + '● Update(~/.gstack/projects/gstack-autoplan-chain-9599im/ceo-plans/2026-09-10-user-dashboard.md)\n ⎿ Added 1 line\n\n', + ' Edit file\n /tmp/foreign.md\n────────\n', + 'Example:\n', '> quoted output\n', '```diff\n 1 +quoted\n```\n', + ]) { const r = replay(); r.viewport = prefix + r.viewport; expect(pick(r)).toBeNull(); } + const priorPanel = replay(); priorPanel.viewport = priorPanel.panel + '\n' + priorPanel.panel; expect(pick(priorPanel)).toBeNull(); +}); + +test('malformed completed-output gutters cannot become a panel delimiter', () => { + for (const prefix of [' 1 +wrong indent\n', ' 0 +zero line\n', ' 9007199254740992 +unsafe line\n', ' 1 +row\n +short wrap\n', ' 1 +row\n -wrong kind\n', ' +only a cropped fragment\n']) { + const r = replay(); r.viewport = prefix + r.panel; expect(pick(r)).toBeNull(); + } +}); + +test('the complete current header, exact target, menu and requested replacement remain binding', () => { + for (const change of [ + (r: ReturnType) => { r.viewport = r.viewport.replace('● Update(~/.gstack/', '● Update(/foreign/'); }, + (r: ReturnType) => { r.viewport = r.viewport.replace(/^ …[^\n]+$/m, ' …projects/sibling/ceo-plans/2026-09-10-user-dashboard.md'); }, + (r: ReturnType) => { r.viewport = r.viewport.replace(' Edit file', ' Read file'); }, + (r: ReturnType) => { r.viewport = r.viewport.replace('❯ 1. Yes', '❯ 2. Yes'); }, + (r: ReturnType) => { r.viewport = r.viewport.replace('3. No', '3. Maybe'); }, + (r: ReturnType) => { r.viewport += '\nDo another action.'; }, + (r: ReturnType) => { r.current.input!.new_string = 'Unrelated replacement'; }, + (r: ReturnType) => { fs.writeFileSync(r.file, 'Unrelated current file'); }, + ]) { const r = replay(); change(r); expect(pick(r)).toBeNull(); } +}); + +test('seen, completed, foreign or superseded native requests cannot borrow the valid panel', () => { + const once = replay(), granted = pick(once)!; + expect(pick(once, new Set([granted.signature]))).toBeNull(); + for (const change of [ + (r: ReturnType) => { const e = r.current; r.context.publicTools.push({ kind: 'result', sessionId: e.sessionId, toolUseId: e.toolUseId, timestamp: new Date(r.context.now).toISOString(), isError: false }); }, + (r: ReturnType) => { r.current.sessionId = 'foreign'; }, + (r: ReturnType) => { r.current.name = 'Write'; }, + (r: ReturnType) => { r.current.input!.file_path = r.file + '.foreign'; }, + (r: ReturnType) => { r.context.publicTools.find(e => e.kind === 'result')!.isError = true; }, + (r: ReturnType) => { const e = structuredClone(r.current); e.toolUseId = 'newer-edit'; r.context.publicTools.push(e); }, + ]) { const r = replay(); change(r); expect(pick(r)).toBeNull(); } +}); + +test('metadata fallback uses the same panel boundary while published inputs stay authoritative', () => { + const r = replay(), current = r.current; + const pending = { source: 'pre_tool_use' as const, tool: 'Edit' as const, sessionId: current.sessionId, toolUseId: current.toolUseId, timestamp: captured.pending.timestamp, file: r.file }; + expect(pendingAutoplanArtifactPermissionInput(r.viewport, { ...r.context, pending }, new Set())).toBeNull(); + // Synthetic missing-publication projection; actual AI request was published. + r.context.publicTools = r.context.publicTools.filter(e => e.toolUseId !== current.toolUseId); + const context = { ...r.context, pending }; + expect(pendingAutoplanArtifactPermissionInput(r.viewport, context, new Set())?.input).toBe('1\r'); + expect(pendingAutoplanArtifactPermissionInput(r.viewport, context, new Set([autoplanArtifactMenuKey(r.viewport)]))).toBeNull(); + expect(pendingAutoplanArtifactPermissionInput(r.viewport, { ...context, viewportCapturedAt: Date.parse(pending.timestamp) - 1 }, new Set())).toBeNull(); + r.viewport = 'Example:\n' + r.viewport; + expect(pendingAutoplanArtifactPermissionInput(r.viewport, context, new Set())).toBeNull(); +}); + +test('the exact prefix fixture and controls select only Autoplan', () => { + for (const file of ['test/autoplan-edit-prefix-ai.test.ts', 'test/fixtures/autoplan-edit-prefix-ai.json']) + expect(selectTests([file], E2E_TOUCHFILES, []).selected).toEqual(['autoplan-chain-pty']); +}); diff --git a/test/ceo-section-choice-ai.test.ts b/test/ceo-section-choice-ai.test.ts new file mode 100644 index 000000000..abc983d71 --- /dev/null +++ b/test/ceo-section-choice-ai.test.ts @@ -0,0 +1,104 @@ +import { expect, test } from 'bun:test'; +import { ceoFirstReviewAUQ, ceoStep0Boundary, nativePlanCallFingerprint, planCountQuestionPhase } from './helpers/claude-pty-runner'; +import captured from './fixtures/ceo-section-choice-ai.json'; +import { E2E_TOUCHFILES } from './helpers/touchfiles-data'; + +function call(index = 4): any { + const source = structuredClone(captured.calls[index]!); + return { sessionId: source.sessionId, toolUseId: source.toolUseId, questions: source.questions, + answered: true, failed: false, unansweredQuestionIndices: [], answeredAt: source.answeredAt, + answers: Object.fromEntries(source.questions.map((q, i) => [q.question, source.answers[i]])) }; +} +const fp = (c: any) => nativePlanCallFingerprint(c, 0, true); +function edit(c: any, change: (text: string) => string) { + const q = c.questions[0], answer = c.answers[q.question]; + q.question = change(q.question); c.answers = { [q.question]: answer }; +} + +test('exact captured section choices start review; preceding actual setup does not', () => { + let started = false; const classified: boolean[] = []; + for (let i = 0; i < captured.calls.length; i++) { + const question = fp(call(i)); + expect(ceoFirstReviewAUQ(question)).toBe(captured.calls[i]!.expectedFirstReview); + const phase = planCountQuestionPhase(question, started, ceoStep0Boundary, ceoFirstReviewAUQ); + started = phase.reviewStarted; classified.push(phase.preReview); + } + expect(classified).toEqual([true, true, true, true, false, false, false, false, false]); +}); + +test('an offered alternative and a quoted historical withdrawal retain current review identity', () => { + const alternative = call(); alternative.answers[alternative.questions[0].question] = alternative.questions[0].options[1].label; + expect(ceoFirstReviewAUQ(fp(alternative))).toBe(true); + const quoted = call(); edit(quoted, s => s + '\nHistorical quote: "This issue has been resolved."'); + expect(ceoFirstReviewAUQ(fp(quoted))).toBe(true); +}); + +test.each([ + ['pending', (c: any) => { c.answered = false; }], + ['failed', (c: any) => { c.failed = true; }], + ['unanswered index', (c: any) => { c.unansweredQuestionIndices = [0]; }], + ['missing session', (c: any) => { c.sessionId = ''; }], + ['missing tool id', (c: any) => { c.toolUseId = ''; }], + ['unoffered answer', (c: any) => { c.answers[c.questions[0].question] = 'Not offered'; }], + ['missing answer', (c: any) => { c.answers = {}; }], + ['mixed packet', (c: any) => { c.questions.push(structuredClone(c.questions[0])); }], + ['multi-select', (c: any) => { c.questions[0].multiSelect = true; }], + ['duplicate options', (c: any) => { c.questions[0].options[1] = structuredClone(c.questions[0].options[0]); }], + ['missing description', (c: any) => { c.questions[0].options[1].description = ''; }], + ['option identity', (c: any) => { c.questions[0].options[1].label = '1B) Other'; }], + ['section mismatch', (c: any) => edit(c, s => s.replace('Section 1 Architecture.', 'Section 2 Architecture.'))], + ['recommendation mismatch', (c: any) => edit(c, s => s.replace('Recommendation: A', 'Recommendation: B'))], + ['missing stakes', (c: any) => edit(c, s => s.replace(/^Stakes if we pick wrong:.*$/m, ''))], + ['duplicate assessment', (c: any) => edit(c, s => s + '\nELI10: A second competing assessment.')], + ['quoted assessment', (c: any) => edit(c, s => s.replace(/^ELI10: (.+)$/m, 'ELI10: "$1"'))], + ['fenced context', (c: any) => edit(c, s => s.replace(/^(Project\/branch\/task:.*)$/m, '```\n$1\n```'))], + ['Suppose assessment', (c: any) => edit(c, s => s.replace('ELI10: The plan', 'ELI10: Suppose the plan'))], + ['single quoted assessment', (c: any) => edit(c, s => s.replace(/^ELI10: (.+)$/m, "ELI10: '$1'"))], + ['current withdrawal', (c: any) => edit(c, s => s + '\nThis issue is withdrawn.')], + ['completed withdrawal', (c: any) => edit(c, s => s + '\nWe have withdrawn this finding.')], + ['conditional assessment', (c: any) => edit(c, s => s.replace('ELI10: The plan', 'ELI10: If the plan'))], + ['withdrawn issue', (c: any) => edit(c, s => s + '\nWe withdraw this finding.')], + ['resolved issue', (c: any) => edit(c, s => s + '\nThis issue has been resolved.')], + ['administrative report', (c: any) => edit(c, s => s.replace(/^.*\n/, '1A — Should the completed review report be saved?\n'))], + ['setup header', (c: any) => { c.questions[0].header = 'Setup'; }], + ['borrowed qid', (c: any) => edit(c, s => s + '\n')], +])('rejects %s despite numbered review prose', (_name, mutate) => { + const c = call(); mutate(c); expect(ceoFirstReviewAUQ(fp(c))).toBe(false); +}); + +test('fingerprints cannot borrow another native call or its options', () => { + const original = fp(call()); + expect(ceoFirstReviewAUQ({ ...original, signature: 'other:tool' })).toBe(false); + expect(ceoFirstReviewAUQ({ ...original, nativeCall: undefined })).toBe(false); + expect(ceoFirstReviewAUQ({ ...original, options: original.options.slice(1) })).toBe(false); +}); + +test('the regression inputs belong to the existing paid CEO case', () => { + expect(E2E_TOUCHFILES['plan-ceo-finding-count']).toContain('test/ceo-section-choice-ai.test.ts'); + expect(E2E_TOUCHFILES['plan-ceo-finding-count']).toContain('test/fixtures/ceo-section-choice-ai.json'); +}); + +test('coherent finished-note destination is administrative, despite matching section and choice', () => { + const c = call(), q = c.questions[0]; + q.header = 'Destination'; + q.question = '1A — Which storage location should hold these notes?\nProject/branch/task: main, Stripe payment webhook plan, Section 1 Architecture.\nELI10: The review is finished; these notes can be saved in either folder for convenience.\nStakes if we pick wrong: People may have to look in a second folder.\nRecommendation: A because the existing folder is easier to find.'; + q.options = [{label:'A) Save beside the plan',description:'Keeps the finished notes together.'},{label:'B) Save in another folder',description:'Keeps finished notes separate.'}]; + c.answers = {[q.question]: q.options[0].label}; + expect(ceoFirstReviewAUQ(fp(c))).toBe(false); +}); + +test('conditional stakes remain valid when the assessment asserts the current gap', () => { + const c = call(); edit(c, s => s.replace('Stakes if we pick wrong:', 'Stakes if we pick wrong: Suppose there were an issue.')); + expect(ceoFirstReviewAUQ(fp(c))).toBe(true); +}); + +test('negated gaps and administrative missing fields cannot borrow review identity', () => { + const negated = call(); edit(negated, s => s.replace(/^ELI10: .+$/m, 'ELI10: The transaction order is not unspecified. The plan guarantees commit before email.')); + expect(ceoFirstReviewAUQ(fp(negated))).toBe(false); + const admin = call(), q = admin.questions[0]; + q.header = 'Destination'; + q.question = '1A — Which storage location should hold these notes?\nProject/branch/task: main, Stripe payment webhook plan, Section 1 Architecture.\nELI10: These finished notes have a missing storage location.\nStakes if we pick wrong: People may look in the wrong folder.\nRecommendation: A because a notes folder is easy to find.'; + q.options = [{label:'A) Add a notes folder',description:'Save the finished notes together.'},{label:'B) Use the existing folder',description:'No new folder.'}]; + admin.answers = {[q.question]: q.options[0].label}; + expect(ceoFirstReviewAUQ(fp(admin))).toBe(false); +}); diff --git a/test/design-first-issue-ai.test.ts b/test/design-first-issue-ai.test.ts new file mode 100644 index 000000000..55980cc5d --- /dev/null +++ b/test/design-first-issue-ai.test.ts @@ -0,0 +1,132 @@ +import { expect, test } from 'bun:test'; +import captured from './fixtures/design-first-issue-ai.json'; +import { isDesignCountFirstReview, isDesignCountSetup } from './helpers/design-count-review'; +import { designStep0Boundary, planCountQuestionPhase } from './helpers/claude-pty-runner'; + +const findings = captured.calls.filter(row => row.ordinal >= 3 && row.ordinal <= 7); +test.each(findings)('actual completed Design D$ordinal starts review', ({ fingerprint }) => { + expect(isDesignCountFirstReview(fingerprint)).toBe(true); + expect(planCountQuestionPhase(fingerprint, false, designStep0Boundary, isDesignCountFirstReview, isDesignCountSetup)).toMatchObject({ preReview: false, reviewStarted: true }); +}); + +function first(): any { return structuredClone(findings[0]!.fingerprint); } +function question(fp: any, text: string) { + const call = fp.nativeCall, old = call.questions[0].question; + call.questions[0].question = text; + call.answers = { [text]: call.answers[old] }; +} +function options(fp: any, change: (q: any) => void) { + const call = fp.nativeCall, q = call.questions[0]; + change(q); + fp.options = q.options.map((o: any, i: number) => ({ index: i + 1, label: o.label })); + call.answers = { [q.question]: q.options[0].label }; +} + +test('actual routing, learnings and future typography TODO do not start a design review', () => { + for (const row of captured.calls.filter(row => [1, 2, 9].includes(row.ordinal))) + expect(isDesignCountFirstReview(row.fingerprint)).toBe(false); +}); + +test('any offered answer and menu order can resolve a substantive finding', () => { + for (const row of findings) { + for (const option of row.fingerprint.nativeCall.questions[0]!.options) { + const fp: any = structuredClone(row.fingerprint); + fp.nativeCall.answers = { [fp.nativeCall.questions[0].question]: option.label }; + expect(isDesignCountFirstReview(fp)).toBe(true); + } + const fp: any = structuredClone(row.fingerprint); + options(fp, q => q.options.reverse()); + expect(isDesignCountFirstReview(fp)).toBe(true); + } +}); + +test('native completion, request identity, current answers and aligned menu are required', () => { + for (const change of [ + (f: any) => { delete f.nativeCall; }, + (f: any) => { f.nativeCall.answered = false; }, + (f: any) => { f.nativeCall.failed = true; }, + (f: any) => { f.nativeCall.sessionId = 'foreign'; }, + (f: any) => { f.nativeCall.toolUseId = 'stale-request'; }, + (f: any) => { f.nativeQuestionIndex = 1; }, + (f: any) => { f.nativeCall.unansweredQuestionIndices = [0]; }, + (f: any) => { f.nativeCall.answers = {}; }, + (f: any) => { f.nativeCall.answers[f.nativeCall.questions[0].question] = 'not offered'; }, + (f: any) => { f.nativeCall.questions[0].question += '\nCorrection: this is a new question.'; }, + (f: any) => { f.nativeCall.answeredAt = 'invalid'; }, + (f: any) => { f.nativeCall.questions.push(structuredClone(f.nativeCall.questions[0])); }, + (f: any) => { f.nativeCall.questions[0].multiSelect = true; }, + (f: any) => { f.options.reverse(); }, + (f: any) => { f.nativeCall.questions[0].header = 'Issue 7'; }, + ]) { const fp = first(); change(fp); expect(isDesignCountFirstReview(fp)).toBe(false); } +}); + +test('numbered issue and all choice identifiers agree without depending on D numbering', () => { + const fp = first(); question(fp, fp.nativeCall.questions[0].question.replace('D3 —', 'D27:')); + expect(isDesignCountFirstReview(fp)).toBe(true); + for (const change of [ + (q: any) => { q.options[0].label = q.options[0].label.replace('1A:', '2A:'); }, + (q: any) => { q.options[1].label = q.options[0].label; }, + ]) { const f = first(); options(f, change); expect(isDesignCountFirstReview(f)).toBe(false); } +}); + +test('a design Issue heading cannot borrow review content for setup, navigation or future work', () => { + for (const title of [ + 'Should we run outside design voices now?', + 'How should we configure design review routing?', + 'What review should run after the design review?', + 'Should we record an app-wide typography TODO?', + 'What type scale will form labels use after a future redesign?', + ]) { + const fp = first(); question(fp, fp.nativeCall.questions[0].question.replace(/Issue 1: [^\n]+/, `Issue 1: ${title}`)); + expect(isDesignCountFirstReview(fp)).toBe(false); + } + for (const replacement of ['PLAN.md onboarding', 'PLAN.md post-review TODO', 'PLAN.md engineering review']) { + const fp = first(); question(fp, fp.nativeCall.questions[0].question.replace('PLAN.md design review', replacement)); + expect(isDesignCountFirstReview(fp)).toBe(false); + } +}); + +test('quoted, hypothetical and withdrawn declarations cannot start the phase', () => { + for (const change of [ + (text: string) => `Example: ${text}`, + (text: string) => `\`\`\`text\n${text}\n\`\`\``, + (text: string) => text.replace('ELI10: ', 'ELI10: Example only: '), + (text: string) => `${text}\nCorrection: that question was hypothetical and is withdrawn.`, + (text: string) => text.replace('How should Save', 'If we later proceed, how should Save'), + ]) { const fp = first(); question(fp, change(fp.nativeCall.questions[0].question)); expect(isDesignCountFirstReview(fp)).toBe(false); } +}); + +test('concrete design conformance and an opposed current violation belong to different offered choices', () => { + for (const change of [ + (q: any) => { q.options.forEach((o: any) => { o.description = 'This is an available option.'; }); }, + (q: any) => { q.options[0].description = '✅ Example only: ' + q.options[0].description; }, + (q: any) => { q.options[2].description = 'No current design gap remains.'; }, + (q: any) => { q.options[0].label = '1A: Run primary review (recommended)'; }, + ]) { const fp = first(); options(fp, change); expect(isDesignCountFirstReview(fp)).toBe(false); } +}); + +import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles'; +test('the new exact public fixture and controls select only the design count workflow', () => { + for (const path of ['test/design-first-issue-ai.test.ts', 'test/fixtures/design-first-issue-ai.json']) + expect(selectTests([path], E2E_TOUCHFILES, []).selected).toEqual(['plan-design-finding-count']); +}); + +test('the assessment asserts a current defect, preserving conditional stakes and quoted history', () => { + for (const change of [ + (s: string) => s.replace('ELI10: The header', 'ELI10: Suppose the header'), + (s: string) => s.replace(/^ELI10: (.+)$/m, "ELI10: '$1'"), + (s: string) => s.replace(/^ELI10: (.+)$/m, 'ELI10: "$1"'), + (s: string) => s.replace('\nStakes if we pick wrong:', ' This issue is withdrawn.\nStakes if we pick wrong:'), + (s: string) => s.replace('\nStakes if we pick wrong:', ' We have resolved this finding.\nStakes if we pick wrong:'), + ]) { const fp = first(); question(fp, change(fp.nativeCall.questions[0].question)); expect(isDesignCountFirstReview(fp)).toBe(false); } + const conditional = first(); question(conditional, conditional.nativeCall.questions[0].question.replace('Stakes if we pick wrong:', 'Stakes if we pick wrong: If we leave this unchanged,')); + expect(isDesignCountFirstReview(conditional)).toBe(true); + const history = first(); question(history, history.nativeCall.questions[0].question.replace('\nStakes if we pick wrong:', ' The old report claimed "We have resolved this finding.", but that claim was wrong.\nStakes if we pick wrong:')); + expect(isDesignCountFirstReview(history)).toBe(true); +}); + +test('an explicit no-current-issue assessment cannot borrow the offered fixes', () => { + const fp = first(); + question(fp, fp.nativeCall.questions[0].question.replace(/^ELI10: .+$/m, 'ELI10: The header shows four clearly differentiated buttons. DESIGN.md is fully followed. No current issue remains.')); + expect(isDesignCountFirstReview(fp)).toBe(false); +}); diff --git a/test/eng-declared-regression-ai.test.ts b/test/eng-declared-regression-ai.test.ts new file mode 100644 index 000000000..5f1e3eee9 --- /dev/null +++ b/test/eng-declared-regression-ai.test.ts @@ -0,0 +1,197 @@ +import { expect, test } from 'bun:test'; +import fixture from './fixtures/eng-declared-regression-ai.json'; +import { evaluateEngSeedCoverage } from './helpers/eng-seeded-coverage'; +import type { PlanCountTranscript } from './helpers/plan-count-transcript'; +import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles'; + +const transcript = fixture.transcript as PlanCountTranscript; +const { start, end } = fixture.provenance.window; +const plan = [ + '## Tests\n\n### CRITICAL regression (mandatory, regression rule)\n\n' + fixture.mandatory, + '## Implementation Tasks\n\n' + fixture.task, + '## Verification\n\n' + fixture.verification, + fixture.reviewReport, +].join('\n\n'); +const evaluate = (text = plan, native = transcript) => evaluateEngSeedCoverage(native, text, start, end); +const retryPlan = [ + '## Architecture\n\n' + fixture.retry.legacyHeading + '\n\n' + fixture.retry.legacy, + '## Tests\n\n' + fixture.retry.heading + '\n\n' + fixture.retry.mandatory, + '## Implementation Tasks\n\n' + fixture.retry.task, + fixture.retry.reviewReport, +].join('\n\n'); +const evaluateRetry = (text = retryPlan) => evaluateEngSeedCoverage(fixture.retry.transcript as PlanCountTranscript, + text, fixture.retry.provenance.window.start, fixture.retry.provenance.window.end); + +test('actual mandatory suite, numbered task and unchanged baseline bind legacy regression', () => { + const result = evaluate(); + expect(Object.keys(result.decisions)).toHaveLength(4); + expect(result.missing).toEqual([]); + expect(result.regression).toBe('plan'); + expect(result.ok).toBe(true); + expect(fixture.provenance.retrospectivePass).toBe(false); +}); + +test('retry same-fixture contract compares new behavior with the unchanged legacy release oracle', () => { + const result = evaluateRetry(); + expect(result.missing).toEqual([]); + expect(result.regression).toBe('plan'); + expect(result.ok).toBe(true); + expect(fixture.retry.provenance.retrospectivePass).toBe(false); +}); + +test('retry requires an unchanged legacy release oracle and actual result parity', () => { + for (const text of [ + retryPlan.replace(fixture.retry.legacy, ''), + retryPlan.replace('stays callable and unchanged this release', 'will be rewritten this release'), + retryPlan.replace('stays callable and unchanged this release', 'might stay callable and unchanged this release'), + retryPlan.replace('- A tenant-keyed flag', 'If approved:\n\n- A tenant-keyed flag'), + retryPlan.replace('- A tenant-keyed flag', 'Unless rejected.\n\n- A tenant-keyed flag'), + retryPlan.replace('- A tenant-keyed flag', 'Proposed baseline:\n\n- A tenant-keyed flag'), + retryPlan.replace('legacyAuthFlow()` stays callable', 'newAuthFlow()` stays callable'), + retryPlan.replace('Run each fixture', 'Run different fixtures'), + retryPlan.replace('and assert identical `Session` shape on success', 'and document different `Session` shape on success'), + retryPlan.replace('identical error code on failure', 'similar error code on failure'), + retryPlan.replace('through `legacyAuthFlow()` and', 'through `newAuthFlow()` and'), + retryPlan.replace('AuthBroker.authenticate()', 'NewBroker.authenticate()'), + retryPlan.replace('and AuthBroker, identical', 'and NewBroker, identical'), + retryPlan.replace('identical Session / error codes', 'identical OtherResponse / error codes'), + retryPlan.replace('Files: src/auth/authFlow.contract.test.ts', 'Files: src/auth/other.contract.test.ts'), + retryPlan.replace('suite green on both paths', 'suite green on the new path'), + retryPlan.replace(fixture.retry.task, ''), + retryPlan.replace('This test\nis also the gate', 'This optional test\nis also the gate'), + ]) { expect(text).not.toBe(retryPlan); expect(evaluateRetry(text).regression, text).toBeUndefined(); } +}); + +test('retry proposals and current withdrawals cannot supply parity coverage', () => { + for (const text of [ + retryPlan.replace(fixture.retry.mandatory, 'If approved, ' + fixture.retry.mandatory), + retryPlan.replace(fixture.retry.mandatory, '"' + fixture.retry.mandatory + '"'), + retryPlan.replace(fixture.retry.mandatory, '```\n' + fixture.retry.mandatory + '\n```'), + retryPlan.replace(fixture.retry.mandatory, fixture.retry.mandatory + '\nThis test is withdrawn.'), + retryPlan.replace(fixture.retry.task, fixture.retry.task + '\nT4 is no longer required.'), + retryPlan.replace(fixture.retry.legacy, fixture.retry.legacy + '\nCorrection: legacyAuthFlow() is changed this release.'), + retryPlan.replace(fixture.retry.legacy, fixture.retry.legacy.split('\n').map(line => '> ' + line).join('\n')), + '# Hypothetical example\n\n' + retryPlan, + '# Proposed work\n\n' + retryPlan, + ]) { expect(text).not.toBe(retryPlan); expect(evaluateRetry(text).regression, text).toBeUndefined(); } +}); + +test('consistent retry identities and unrelated negative outcomes retain parity evidence', () => { + for (const text of [ + retryPlan.replaceAll('AuthBroker', 'SessionBroker').replaceAll('Session', 'Reply'), + retryPlan.replaceAll('authFlow.contract.test.ts', 'loginFlow.contract.test.js').replaceAll('T4', 'T14'), + retryPlan.replace(fixture.retry.task, fixture.retry.task + '\n - Verify revoked tokens are rejected.'), + retryPlan.replace(fixture.retry.legacy, fixture.retry.legacy + '\nRejected alternatives stay documented.'), + ]) expect(evaluateRetry(text).regression).toBe('plan'); +}); + +for (const prefix of [ + '# Source\n\n', + 'An unproven hypothesis.\n\n', + 'The following is a hypothetical example.\n\n', + 'The following is source material only, not the current reviewed plan.\n\n', + '# Current reviewed plan\n\nThe following sections reproduce source material only; they are not requirements of this plan.\n\n', +]) { + test('first and retry evidence retain enclosing source frame: ' + prefix.trim(), () => { + expect(evaluate(prefix + plan).regression).toBeUndefined(); + expect(evaluateRetry(prefix + retryPlan).regression).toBeUndefined(); + }); +} + +test('declaration wording and task identity can vary without changing the required baseline', () => { + for (const text of [ + plan.replaceAll('T4', 'T12'), + plan.replace('capture current', 'pin existing'), + plan.replace('is added as a critical', 'is required as a mandatory'), + plan.replace('auth/legacy tests — ', 'core/auth — '), + plan.replace('wrong-audience, wrong-issuer,', 'wrong-audience, wrong-issuer, malformed,'), + plan.replace(fixture.task, fixture.task + '\n - Verify expired and revoked tokens are rejected.'), + plan.replace(fixture.mandatory, fixture.mandatory + '\nKeep a record of rejected alternatives.'), + '# Historical example\n\nA proposed suite was discussed.\n\n# Current reviewed plan\n\n' + plan, + ]) expect(evaluate(text).regression, text).toBe('plan'); +}); + +test('declaration, task, target and original baseline cannot lend each other missing evidence', () => { + for (const text of [ + plan.replace(fixture.mandatory, ''), + plan.replace(fixture.task, ''), + plan.replace(fixture.verification, ''), + plan.replace('suite (T4)', 'suite (T5)'), + plan.replace('against the untouched', 'against the rewritten'), + plan.replace('first and commit it green. This is the baseline.', 'after rollout and document it.'), + plan.replace('capture current', 'describe future'), + plan.replace('is\nthe oracle the new path is compared to', 'is documentation the new path links to'), + plan.replace('characterization test suite** for `legacyAuthFlow()`', 'characterization test suite** for `newAuthFlow()`'), + plan.replace('suite for `legacyAuthFlow()` prior behavior', 'suite for `newAuthFlow()` prior behavior'), + plan.replace('untouched `legacyAuthFlow()`', 'untouched `newAuthFlow()`'), + ]) { expect(text).not.toBe(plan); expect(evaluate(text).regression, text).toBeUndefined(); } +}); + +test('proposals, future work, conditional and quoted declarations are not required coverage', () => { + for (const text of [ + plan.replace('is added as a critical', 'will be added as a critical'), + plan.replace('is added as a critical', 'might be added as a critical'), + plan.replace('is added as a critical', 'is not added as a critical'), + plan.replace(fixture.mandatory, 'If approved, ' + fixture.mandatory), + plan.replace(fixture.mandatory, 'Example: ' + fixture.mandatory), + plan.replace(fixture.mandatory, 'An unproven hypothesis. ' + fixture.mandatory), + plan.replace(fixture.mandatory, '"' + fixture.mandatory + '"'), + plan.replace(fixture.mandatory, "'" + fixture.mandatory + "'"), + plan.replace(fixture.mandatory, fixture.mandatory.split('\n').map(line => '> ' + line).join('\n')), + plan.replace(fixture.mandatory, '```\n' + fixture.mandatory + '\n```'), + '# Hypothetical example\n\n' + plan, + '# Quoted source\n\n' + plan, + '# Proposed work\n\n' + plan, + plan.replace('## Verification\n\n1. Run', '## Verification\n\n1. If approved, run'), + ]) { expect(text).not.toBe(plan); expect(evaluate(text).regression, text).toBeUndefined(); } +}); + +test('withdrawal of the owned suite, task or baseline prevents credit', () => { + for (const [from, addition] of [ + [fixture.mandatory, 'This suite is withdrawn.'], + [fixture.mandatory, 'The characterization suite is not required.'], + [fixture.mandatory, 'Do not run the suite.'], + [fixture.task, 'Correction: T4 is cancelled.'], + [fixture.task, 'This task is deferred.'], + [fixture.verification, 'Correction: T4 is cancelled.'], + [fixture.verification, 'Skip the characterization suite.'], + ]) expect(evaluate(plan.replace(from!, from + '\n' + addition)).regression, addition).toBeUndefined(); +}); + +test('the required suite cannot replace completed distinct native decisions or final report', () => { + for (let index = 0; index < transcript.calls.length; index++) { + const native = structuredClone(transcript); + native.calls.splice(index, 1); + expect(evaluate(plan, native).ok).toBe(false); + expect(evaluate(plan, native).missing).toHaveLength(1); + } + for (const mutate of [ + (native: PlanCountTranscript) => { native.calls[0]!.failed = true; }, + (native: PlanCountTranscript) => { native.calls[0]!.answeredAt = new Date(start - 1).toISOString(); }, + (native: PlanCountTranscript) => { native.calls[0]!.sessionId = 'foreign-session'; }, + (native: PlanCountTranscript) => { native.calls.push(structuredClone(native.calls[0]!)); }, + ]) { + const native = structuredClone(transcript); mutate(native); + expect(evaluate(plan, native).ok).toBe(false); + } + expect(evaluate(plan.replace(fixture.reviewReport, '')).problems).toContain('final review report absent or empty'); +}); + +test('public declaration still requires the existing owned time and session interval', () => { + const native = structuredClone(transcript); + native.assistantMessages = [{ sessionId: native.calls[0]!.sessionId, timestamp: new Date(start).toISOString(), text: plan }]; + expect(evaluate(fixture.reviewReport, native).regression).toBe('public-narration'); + native.assistantMessages[0]!.timestamp = new Date(start - 1).toISOString(); + expect(evaluate(fixture.reviewReport, native).regression).toBeUndefined(); + native.assistantMessages[0]!.timestamp = new Date(end + 1).toISOString(); + expect(evaluate(fixture.reviewReport, native).regression).toBeUndefined(); + native.assistantMessages[0]!.timestamp = new Date(start).toISOString(); + native.assistantMessages[0]!.sessionId = 'foreign-session'; + expect(evaluate(fixture.reviewReport, native).regression).toBeUndefined(); +}); + +test('new declaration evidence registers only the two existing engineering count owners', () => { + for (const file of ['test/eng-declared-regression-ai.test.ts', 'test/fixtures/eng-declared-regression-ai.json']) { + expect(selectTests([file], E2E_TOUCHFILES, []).selected.sort()).toEqual(['plan-eng-finding-count', 'plan-eng-multi-finding-batching']); + } +}); diff --git a/test/eng-seeded-completion-ai.test.ts b/test/eng-seeded-completion-ai.test.ts new file mode 100644 index 000000000..380557250 --- /dev/null +++ b/test/eng-seeded-completion-ai.test.ts @@ -0,0 +1,177 @@ +import { expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import fixture from './fixtures/eng-seeded-completion-ai.json'; +import { classifyVisible, extractPlanFilePath } from './helpers/claude-pty-runner'; +import * as predicates from './helpers/claude-pty-runner'; +import { selectTests, E2E_TOUCHFILES } from './helpers/touchfiles'; + +const gate = '─────\nClaude has written up a plan and is ready to execute. Would you like to proceed?\n❯ 1. Yes, and use auto mode\n2. Yes, manually approve edits\n3. Tell Claude what to change'; +const compactGate = 'Exit plan mode?\nClaude wants to exit plan mode\n❯ 1. Yes, and switch to default (ask each time) for this session\n2. No'; +const question = 'Which runner should the plan use?\nA) Use the built-in runner\nB) Build a custom runner\nRecommendation: A because it avoids duplicate scheduling logic.\nReply with A or B.'; +const classify = (history: string, currentScreen: string) => classifyVisible(history, { strictPlanWrites: true, currentScreen }); + +test('historical TODO excerpt supplies no current seeded completion or plan file', () => { + expect(classifyVisible(fixture.visibleReadExcerpt, { strictPlanWrites: true })?.outcome).toBe('plan_ready'); + expect(extractPlanFilePath(fixture.visibleReadExcerpt)).toBeNull(); + expect(classify(fixture.visibleReadExcerpt, fixture.visibleReadExcerpt)).toBeNull(); + expect(classify(fixture.visibleReadExcerpt, fixture.reconstructedScreen)).toBeNull(); + expect(classify(fixture.visibleReadExcerpt, '')).toBeNull(); +}); + +test('only a complete current native approval panel establishes seeded plan_ready', () => { + for (const current of [gate, compactGate]) { + expect(classify(fixture.visibleReadExcerpt + '\n' + current, current)?.outcome).toBe('plan_ready'); + for (const invalid of [ + '', 'Still reviewing the draft.', current + '\nStill reviewing the draft.', + current.split('\n').slice(0, -1).join('\n'), current.replace('❯', ''), + current.replace(/2\.[^\n]+/, '2. Approve another action'), + 'Example:\n' + current, '```text\n' + current, current.split('\n').map(line => '> ' + line).join('\n'), + ]) expect(classify(fixture.visibleReadExcerpt + '\n' + current, invalid), invalid).toBeNull(); + } +}); + +test('ignoring old completion text preserves a genuine current question and stronger failure outcomes', () => { + for (const history of [fixture.visibleReadExcerpt, gate, compactGate]) { + expect(classify(history + '\n' + question, question)?.outcome).toBe('asked'); + } + expect(classify(fixture.visibleReadExcerpt + '\n' + question, fixture.visibleReadExcerpt + '\n' + question)?.outcome).toBe('asked'); + expect(classify('⏺ Write(/tmp/.claude/plans/review.md)\n' + gate, gate)?.outcome).toBe('wrote_findings_before_asking'); + expect(classify('⏺ Write(/tmp/implementation.ts)\n' + fixture.visibleReadExcerpt, '')?.outcome).toBe('silent_write'); + // Callers which do not opt into the current viewport retain their contract. + expect(classifyVisible('The item is ready to execute.')?.outcome).toBe('plan_ready'); + expect(classifyVisible(question)?.outcome).toBe('asked'); +}); + +test('real PTY waits past old TODO, stale, partial and mismatched panels but accepts the current gate', async () => { + const scenarios = [ + { name: 'todo', initial: fixture.visibleReadExcerpt, expected: 'asked' }, + { name: 'stale', initial: gate + '\u001b[2J\u001b[HStill reviewing the draft.', expected: 'asked' }, + { name: 'partial', initial: gate.split('\n').slice(0, -1).join('\n'), expected: 'asked' }, + { name: 'mismatch', initial: gate.replace('3. Tell Claude what to change', '3. Delete the draft'), expected: 'asked' }, + { name: 'complete', initial: gate, expected: 'plan_ready' }, + { name: 'cursorless-timeout', initial: gate.replace('❯ ', ''), expected: 'timeout' }, + ]; + const results = await Promise.allSettled(scenarios.map(async scenario => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'seeded-completion-')); + const cli = path.join(dir, 'fake-claude'), working = path.join(dir, 'repo'); + fs.mkdirSync(working); + fs.writeFileSync(cli, `#!/usr/bin/env bun +const fs = require('node:fs'); +fs.writeFileSync(process.env.COMPLETION_ARGV, JSON.stringify(process.argv.slice(2))); +let sent = false; +const render = text => process.stdout.write('\\x1b[2J\\x1b[H' + text.replace(/\\n/g, '\\r\\n')); +process.stdin.on('data', chunk => { + if (sent || !chunk.toString().includes('/plan-eng-review')) return; + sent = true; + fs.writeFileSync(process.env.COMPLETION_PHASE, 'initial'); + render(${JSON.stringify(scenario.initial)}); + if (${JSON.stringify(scenario.expected)} === 'asked') setTimeout(() => { + fs.writeFileSync(process.env.COMPLETION_PHASE, 'question'); + render(${JSON.stringify(question)}); + }, 4500); +}); +setInterval(() => {}, 1000); +`, { mode: 0o755 }); + try { + const runner = pathToFileURL(path.join(import.meta.dir, 'helpers/claude-pty-runner.ts')).href; + const childFile = path.join(dir, 'observe.ts'); + fs.writeFileSync(childFile, `import { runPlanSkillObservation, resolveClaudeBinary } from ${JSON.stringify(runner)}; +if (resolveClaudeBinary() !== process.env.BROWSE_TERMINAL_BINARY) throw new Error('Fake CLI resolution failed'); +const obs = await runPlanSkillObservation({ skillName: 'plan-eng-review', inPlanMode: true, + initialPlanContent: '# Plan: Completion regression\\n\\nReview the existing draft.', + cwd: ${JSON.stringify(working)}, timeoutMs: 12000, + env: { COMPLETION_ARGV: process.env.COMPLETION_ARGV, COMPLETION_PHASE: process.env.COMPLETION_PHASE } }); +console.log(JSON.stringify(obs)); +`); + // Only this isolated child receives the executable override; it verifies + // the resolver before the real PTY launch, so no provider can be invoked. + const child = Bun.spawn([process.execPath, childFile], { cwd: process.cwd(), + env: { ...process.env, BROWSE_TERMINAL_BINARY: cli, + COMPLETION_ARGV: path.join(dir, 'argv.json'), COMPLETION_PHASE: path.join(dir, 'phase.txt'), + EVALS_RUN_ID: 'seeded-completion-fake', GSTACK_EVAL_DIR: path.join(dir, 'evidence') }, + stdout: 'pipe', stderr: 'pipe' }); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited, + ]); + expect(exitCode, stderr).toBe(0); + const obs = JSON.parse(stdout.trim().split('\n').at(-1)!); + expect(obs.outcome, scenario.name).toBe(scenario.expected); + expect(fs.readFileSync(path.join(dir, 'phase.txt'), 'utf8'), scenario.name).toBe(scenario.expected === 'asked' ? 'question' : 'initial'); + expect(obs.planFile).toBeUndefined(); + expect(obs.scopeGateAutoSelectObserved).toBe(false); + const args = JSON.parse(fs.readFileSync(path.join(dir, 'argv.json'), 'utf8')); + expect(args.filter((arg: string) => arg === '--session-id')).toHaveLength(1); + expect(args).toContain('--permission-mode'); expect(args).toContain('plan'); + const saved = JSON.parse(fs.readFileSync(path.join(obs.artifactDir, 'observation.json'), 'utf8')); + expect(saved.scopeSessionId).toBe(args[args.indexOf('--session-id') + 1]); + expect(saved.scopeGateAutoSelectObserved).toBe(false); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } + })); + for (let i = 0; i < results.length; i++) { + const result = results[i]!; + expect(result.status, `${scenarios[i]!.name}: ${result.status === 'rejected' ? String(result.reason) : 'complete'}`).toBe('fulfilled'); + } +}, 45000); + +async function mockedObservation(frames: string[], verdict: 'waiting' | 'working', seeded = true) { + // Execute the unchanged observer function with its real classifiers, a + // synthetic clock/session, and a stubbed judge. No CLI or judge is launched. + const source = fs.readFileSync(path.join(import.meta.dir, 'helpers/claude-pty-runner.ts'), 'utf8'); + const start = source.indexOf('export async function runPlanSkillObservation('); + const end = source.indexOf('\n// ─', start); + expect(start).toBeGreaterThan(0); expect(end).toBeGreaterThan(start); + const executable = source.slice(start, end).replace('export async function', 'async function') + '\nreturn runPlanSkillObservation;'; + const js = new Bun.Transpiler({ loader: 'ts' }).transformSync(executable); + let clock = 0, tick = -1, closed = 0, judged = 0; + const current = () => frames[Math.min(Math.max(tick, 0), frames.length - 1)]!; + const args: Record = { + path, process: { cwd: () => '/synthetic-owned' }, Date: { now: () => clock }, randomUUID: () => 'owned', + Bun: { sleep: async (ms: number) => { if (ms === 2000) { tick++; clock += tick === 0 && frames.length > 1 ? 2000 : 61000; } else clock += ms; } }, + launchClaudePty: async () => ({ send: () => {}, mark: () => 0, exited: () => false, + visibleSince: current, rawOutput: current, currentScreen: async () => current(), hermeticConfigDir: null, + close: async () => { closed++; } }), + createPlanCountSnapshotWriter: () => () => ({}), logPtySnapshot: () => {}, + isProseAUQVisible: predicates.isProseAUQVisible, isPlanReadyVisible: predicates.isPlanReadyVisible, + isScopeGateQuestionVisible: predicates.isScopeGateQuestionVisible, + isScopeGateAutoSelectVisible: predicates.isScopeGateAutoSelectVisible, + classifyVisible, extractPlanFilePath, findNativeAutoDecision: () => null, + judgePtyState: () => { judged++; return { state: verdict, reasoning: 'synthetic current-frame verdict' }; }, + }; + const run = new Function(...Object.keys(args), js)(...Object.values(args)); + const obs = await run({ skillName: 'plan-eng-review', timeoutMs: 70000, + ...(seeded ? { initialPlanContent: '# Plan: Required draft' } : {}) }); + expect(closed).toBe(1); + return { obs, judged }; +} + +for (const [name, current] of [ + ['cursorless approval', gate.replace('❯ ', '')], + ['partial approval', gate.split('\n').slice(0, -1).join('\n')], +] as const) test(`rejected seeded ${name} cannot gain prose or judge waiting credit`, async () => { + const { obs, judged } = await mockedObservation([current], 'waiting'); + expect(judged).toBeGreaterThan(0); + expect(obs.outcome).toBe('timeout'); + expect(obs.proseAUQEverObserved).toBe(false); expect(obs.waitingEverObserved).toBe(false); +}); + +test('rejected completion does not erase a genuine earlier question or change unseeded behavior', async () => { + const prior = question + '\nDo you want to create draft.md?\n❯ 1. Yes\n2. No\nEsc to cancel · Tab to amend'; + expect(predicates.isProseAUQVisible(prior)).toBe(true); + expect(classifyVisible(prior)).toBeNull(); + const { obs } = await mockedObservation([prior, gate.replace('❯ ', '')], 'working'); + expect(obs.outcome).toBe('asked'); expect(obs.proseAUQEverObserved).toBe(true); + expect(obs.waitingEverObserved).toBe(false); + const unseeded = await mockedObservation([gate.replace('❯ ', '')], 'waiting', false); + expect(unseeded.obs.outcome).toBe('plan_ready'); expect(unseeded.judged).toBe(0); +}); + +test('completion evidence dependencies select exactly the seeded observation owners', () => { + const owners = ['plan-ceo-review-plan-mode', 'plan-eng-review-plan-mode', 'plan-design-review-plan-mode', + 'plan-devex-review-plan-mode', 'plan-mode-no-op', 'auto-decide-preserved', 'conductor-prose'].sort(); + for (const file of ['test/eng-seeded-completion-ai.test.ts', 'test/fixtures/eng-seeded-completion-ai.json']) { + expect(selectTests([file], E2E_TOUCHFILES).selected.sort()).toEqual(owners); + } +}); diff --git a/test/fixtures/auto-decide-retry-ai.json b/test/fixtures/auto-decide-retry-ai.json new file mode 100644 index 000000000..eab37b4c4 --- /dev/null +++ b/test/fixtures/auto-decide-retry-ai.json @@ -0,0 +1,454 @@ +{ + "transcript": { + "status": "ready", + "calls": [], + "assistantMessages": [ + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "text": "I'll invoke the `/plan-ceo-review` skill against the draft plan.", + "timestamp": "2026-09-10T04:21:58.556Z" + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "text": "Running the preamble first, then a light system audit so the mode question is grounded in the actual listing code.", + "timestamp": "2026-09-10T04:22:10.706Z" + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "text": "Found the relevant code paths. Running the question-tuning check for the mode question and handling the onboarding markers, then I'll present the mode decision.", + "timestamp": "2026-09-10T04:23:41.654Z" + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "text": "Heads-up from the preamble: unshipped work on this branch, so `/review` then `/ship` when you're ready. Also, gstack follows the **Boil the Ocean** principle: do the complete thing when AI makes the marginal cost near zero. Read more at https://garryslist.org/posts/boil-the-ocean if you'd like.\n\n**Review mode: HOLD SCOPE.**\n\nAuto-decided \"Select review mode\" \u2192 HOLD SCOPE (your saved preference on `plan-ceo-review-mode`). Change with /plan-tune.\n\nWhy this mode fits the draft:\n\n- **It is a correctness fix to an existing path, not a new capability.** Skill listings should not change order between machines or filesystems. The context default for bug fixes and refactors is HOLD SCOPE.\n- **The draft already states its invariants.** Membership, aliases, metadata and text/JSON formats stay unchanged. The review's job is to make those invariants bulletproof, not to widen them.\n- **The codebase confirms the problem is real.** The census helper enumerates skills with no sort at all. The context-bill code sorts with the default code-point comparator, so a mixed-case name like `Zeta` lands before `alpha`. Both are places the deep review should pin with tests.\n\nWhat I deliberately left open for the later invocation, at your request:\n\n1. **The implementation approach decision** (a shared case-insensitive comparator versus per-site sorts). The skill requires your explicit approval there before the deep review.\n2. **Which listing path the draft means.** The census helper, the context-bill CLI, or both. This changes the file list.\n3. **Onboarding prompts** (continuous checkpoint feature, cross-project learnings) and the Office Hours offer. Their markers are untouched, so they will fire again next healthy run.\n\nThe plan file at `.claude/plans/please-review-the-following-melodic-kernighan.md` records the mode, the audit findings with file and line references, and the three open items. I have not called ExitPlanMode since the review workflow is paused rather than complete.\n\nSTATUS: DONE. No durable learnings this session.", + "timestamp": "2026-09-10T04:24:41.750Z" + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "text": "I'll invoke the /plan-ceo-review skill against your draft plan.", + "timestamp": "2026-09-10T04:32:12.533Z" + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "text": "Running the skill preamble and base-branch detection first.", + "timestamp": "2026-09-10T04:32:27.444Z" + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "text": "Base branch is `main`. Checking your question-tuning preference for the mode question before I render it.", + "timestamp": "2026-09-10T04:33:00.854Z" + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "text": "Heads-up from gstack: there is unshipped work on this branch, so `/review` then `/ship` when you get to it.\n\n**D1 \u2014 Review mode for the deterministic skill-list ordering draft**\n\nAuto-decided \"Review mode: push scope up, cherry-pick expansions, hold scope, or cut to minimum?\" \u2192 **HOLD SCOPE** (your preference). Change with /plan-tune.\n\nWhy HOLD SCOPE is the recommendation the auto-decide landed on:\n\n- **The draft is a determinism fix to an existing path,** not a new capability. The skill's context defaults map bug fixes and refactors to HOLD SCOPE.\n- **The draft already draws hard boundaries.** Membership, aliases, metadata, and text/JSON formats stay unchanged. That is a hold-scope posture written into the plan itself.\n- **The value is in rigor, not ambition.** The interesting questions are locale-independent comparison for mixed-case names, whether the sort happens once at the listing path or leaks into other enumerators, and how the tests force differing directory order. Those are HOLD SCOPE questions.\n\nSkipped in this invocation, per your request:\n\n- **Office Hours offer and onboarding prompts.** The preamble fired two one-time setup prompts. I did not answer or mark them, so they re-fire on your next healthy run.\n- **Approach selection (0C-bis).** The skill normally wants an approved implementation approach before mode selection. Since you asked for only the mode decision, this is still pending. Expect it as the first question when you run the substantive review.\n- **The 11-section deep review and plan file.** Not started. No plan file was written, so there is no review report and I did not exit plan mode.\n\nNo durable learnings this session.\n\nNext: when you are ready, re-run `/plan-ceo-review` with the draft. Tell it \"HOLD SCOPE already selected\" so it skips D1 and goes straight to approach selection and the deep review.", + "timestamp": "2026-09-10T04:33:46.197Z" + } + ] + }, + "tools": [ + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:22:00.757Z", + "toolUseId": "toolu_01D1z8pVbfkEeHZGgi6JdPZk", + "kind": "use", + "name": "Skill", + "input": { + "skill": "plan-ceo-review", + "args": "Draft: deterministic skill-list ordering. Users compare skill listings in scripts and reviews. Make the existing listing path sort registered skill names deterministically before rendering them. Keep skill membership, aliases, metadata and text/JSON output formats unchanged. Cover mixed-case names and differing directory enumeration order with tests. Review-mode decision only for this invocation." + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:22:02.528Z", + "toolUseId": "toolu_01D1z8pVbfkEeHZGgi6JdPZk", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:22:12.591Z", + "toolUseId": "toolu_01B89LH3gWatoKLbFjkW1duv", + "kind": "use", + "name": "Bash", + "input": { + "command": "_SS=\"$HOME/.claude/skills/gstack/bin/gstack-skill-start\"\n[ -x \"$_SS\" ] || _SS=\".claude/skills/gstack/bin/gstack-skill-start\"\n\"$_SS\" --skill \"plan-ceo-review\" --model \"claude\" --parent-pid \"$PPID\" \\\n || echo \"SKILL_START: unavailable \u2014 stale install; run ./setup or /gstack-upgrade (preamble degraded, continue the user's task)\"", + "description": "Run gstack skill-start preamble" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:22:15.395Z", + "toolUseId": "toolu_01B89LH3gWatoKLbFjkW1duv", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:22:42.272Z", + "toolUseId": "toolu_011L1hSC8vSZLDVGpfiokh1h", + "kind": "use", + "name": "Bash", + "input": { + "command": "git remote get-url origin 2>/dev/null; echo \"---LOG\"; git log --oneline -15; echo \"---STAT\"; git diff main --stat 2>/dev/null | tail -5; echo \"---STASH\"; git stash list | head; echo \"---DESIGN\"; SLUG=$(~/.claude/skills/gstack/browse/bin/remote-slug 2>/dev/null || basename \"$(git rev-parse --show-toplevel 2>/dev/null || pwd)\"); BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null | tr '/' '-'); echo \"SLUG=$SLUG BRANCH=$BRANCH\"; ls -t ~/.gstack/projects/$SLUG/*-design-*.md 2>/dev/null | head -1; ls -t docs/designs/*.md 2>/dev/null | head -3; ls -t ~/.gstack/projects/$SLUG/*-$BRANCH-ceo-handoff-*.md 2>/dev/null | head -1 || echo NO_HANDOFF; echo \"---LEARN\"; ~/.claude/skills/gstack/bin/gstack-config get cross_project_learnings 2>/dev/null || echo unset; ~/.claude/skills/gstack/bin/gstack-learnings-search --limit 5 2>/dev/null | head -20", + "description": "System audit: remote, log, diff, stash, design doc, learnings" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:22:43.881Z", + "toolUseId": "toolu_011L1hSC8vSZLDVGpfiokh1h", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:22:54.173Z", + "toolUseId": "toolu_01Jpe19C6Qy7iLjhxsVVFZEM", + "kind": "use", + "name": "Bash", + "input": { + "command": "cd /home/vercel-sandbox/gstack; ls bin | grep -i -E \"skill|list|catalog|census|route\" ; echo \"---GREP\"; grep -rl -i -E \"list.*skills|skills.*list|listSkills|skill-list|skills list\" --include=*.ts --include=*.sh --include=*.tmpl bin scripts lib _gstack-command setup 2>/dev/null | grep -v node_modules | head -30; echo \"---TODOS\"; grep -n -i -E \"sort|order|determin|listing\" TODOS.md | head -20", + "description": "Locate skill listing code and related TODOs" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:22:54.300Z", + "toolUseId": "toolu_01Jpe19C6Qy7iLjhxsVVFZEM", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:06.413Z", + "toolUseId": "toolu_013RE2TjYQMdqe8nLecHC4vw", + "kind": "use", + "name": "Bash", + "input": { + "command": "cd /home/vercel-sandbox/gstack; grep -n -E '\"(skill:check|skill:list|skills|list|catalog|context-bill)[^\"]*\":' package.json; echo \"---REGISTRY\"; grep -rn -i -E \"registered skill|registeredSkills|skill registry|SKILL_REGISTRY|listRegistered|registerSkill\" --include=*.ts --include=*.sh scripts lib bin test/helpers 2>/dev/null | grep -v node_modules | head -15; echo \"---READDIR\"; grep -rn -E \"readdirSync\\(\" scripts/gen-skill-docs.ts scripts/skill-check.ts scripts/skill-health.ts test/helpers/skill-census.ts lib/*.ts 2>/dev/null | head -20; echo \"---JSON\"; grep -rln -E \"\\-\\-json\" bin scripts --include=* 2>/dev/null | head -20", + "description": "Find skill enumeration and JSON-listing code" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:06.508Z", + "toolUseId": "toolu_013RE2TjYQMdqe8nLecHC4vw", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:13.375Z", + "toolUseId": "toolu_013Ee9LJJQ8R1fR2sQFPGwkj", + "kind": "use", + "name": "Read", + "input": { + "file_path": "/home/vercel-sandbox/gstack/test/helpers/skill-census.ts", + "offset": 40, + "limit": 50 + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:13.401Z", + "toolUseId": "toolu_013Ee9LJJQ8R1fR2sQFPGwkj", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:14.737Z", + "toolUseId": "toolu_01BU2xpwy4uMyx1Xp4quu9bj", + "kind": "use", + "name": "Read", + "input": { + "file_path": "/home/vercel-sandbox/gstack/lib/context-bill.ts", + "offset": 195, + "limit": 30 + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:14.759Z", + "toolUseId": "toolu_01BU2xpwy4uMyx1Xp4quu9bj", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:16.384Z", + "toolUseId": "toolu_013aumciUHw2Bqjaaq27g1EV", + "kind": "use", + "name": "Bash", + "input": { + "command": "cd /home/vercel-sandbox/gstack; grep -n -E \"json|text|format|\\.sort\\(|localeCompare\" lib/context-bill.ts | head -30; echo \"---BIN\"; grep -n -E \"json|sort|format\" bin/gstack-context-bill | head -15; echo \"---SKILLCHECK\"; grep -n -E \"json|\\.sort\\(\" scripts/skill-check.ts | head", + "description": "Check output formats and sorting in listing candidates" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:16.450Z", + "toolUseId": "toolu_013aumciUHw2Bqjaaq27g1EV", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:26.286Z", + "toolUseId": "toolu_015mJBMXZtpHKA6hWxDwTo8j", + "kind": "use", + "name": "Read", + "input": { + "file_path": "/home/vercel-sandbox/gstack/lib/context-bill.ts", + "offset": 335, + "limit": 50 + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:26.306Z", + "toolUseId": "toolu_015mJBMXZtpHKA6hWxDwTo8j", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:28.306Z", + "toolUseId": "toolu_01H7ZHzRW7U18u8zh3L6yKFH", + "kind": "use", + "name": "Bash", + "input": { + "command": "cd /home/vercel-sandbox/gstack; grep -n -E \"json|JSON|renderText|function render|function format\" bin/gstack-context-bill.ts bin/gstack-context-bill 2>/dev/null | head -20; ls bin | grep context-bill; echo \"---TESTS\"; ls test | grep -i -E \"context-bill|census|catalog\"; echo \"---GBRAIN\"; grep -n -E \"\\.skills\\b|skills:\" lib/context-bill.ts | head -10", + "description": "Find context-bill CLI output formats and tests" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:28.376Z", + "toolUseId": "toolu_01H7ZHzRW7U18u8zh3L6yKFH", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:45.519Z", + "toolUseId": "toolu_01BHQiS1mU8LZfxsWwFbGADs", + "kind": "use", + "name": "Bash", + "input": { + "command": "cd /home/vercel-sandbox/gstack; grep -n -i -E \"plan-ceo-review.*(mode|scope)|ceo.*mode\" ~/.claude/skills/gstack/scripts/question-registry.ts | head -5; echo \"---CHECK\"; printf '%s' \"Select review mode (expansion/selective/hold/reduction) for deterministic skill-list ordering plan\" | ~/.claude/skills/gstack/bin/gstack-question-preference --check \"plan-ceo-review-mode-selection\" --summary-stdin 2>&1 | head -5; echo \"---MARKERS\"; touch \"/tmp/gstack-paid-shard-pNXtSA/tmp/gstack-auto-decide-w2UU3e/.completeness-intro-seen\" && echo \"lake-intro marker touched\"", + "description": "Question-tuning check and lake-intro marker" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:47.061Z", + "toolUseId": "toolu_01BHQiS1mU8LZfxsWwFbGADs", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:53.288Z", + "toolUseId": "toolu_01Fz4MuCGi9tb8sPCq6SswEB", + "kind": "use", + "name": "Bash", + "input": { + "command": "cd /home/vercel-sandbox/gstack; printf '%s' \"Select review mode (expansion/selective/hold/reduction) for deterministic skill-list ordering plan\" | ~/.claude/skills/gstack/bin/gstack-question-preference --check \"plan-ceo-review-mode\" --summary-stdin 2>&1 | head -3", + "description": "Re-run tuning check with registered question id" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:54.770Z", + "toolUseId": "toolu_01Fz4MuCGi9tb8sPCq6SswEB", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:24:12.317Z", + "toolUseId": "toolu_01YHaTDadr9x4vFfuus4vamb", + "kind": "use", + "name": "Bash", + "input": { + "command": "cd /home/vercel-sandbox/gstack; ~/.claude/skills/gstack/bin/gstack-question-log '{\"skill\":\"plan-ceo-review\",\"question_id\":\"plan-ceo-review-mode\",\"question_summary\":\"Select review mode for deterministic skill-list ordering plan\",\"category\":\"routing\",\"door_type\":\"two-way\",\"options_count\":4,\"user_choice\":\"C\",\"recommended\":\"C\",\"session_id\":\"634003-1789014135-1db98f5a\"}' 2>/dev/null && echo \"question logged\" || echo \"question-log unavailable\"; ~/.claude/skills/gstack/bin/gstack-skill-end --skill \"plan-ceo-review\" --outcome success --session-id \"634003-1789014135-1db98f5a\" --tel-start \"1789014135\" --used-browse no --error-message \"\" --failed-step \"\" 2>/dev/null && echo \"telemetry ok\" || echo \"telemetry skipped\"", + "description": "Log auto-decided mode question and run skill-end telemetry" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:24:14.288Z", + "toolUseId": "toolu_01YHaTDadr9x4vFfuus4vamb", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:24:26.317Z", + "toolUseId": "toolu_01EVSYPhAqVAdZhoeTZ9bU1r", + "kind": "use", + "name": "Write", + "input": { + "file_path": "/tmp/gstack-paid-shard-pNXtSA/tmp/gstack-hermetic-633854-8JWnVY/with-skills/.claude/plans/please-review-the-following-melodic-kernighan.md", + "content": "# Plan: deterministic skill-list ordering (CEO review \u2014 mode decision only)\n\n## Context\n\nUsers compare skill listings in scripts and reviews. Today the ordering of\nregistered skill names depends on directory enumeration order and, where a\nsort exists, on JavaScript's default code-point sort (uppercase before\nlowercase). The draft plan makes the existing listing path sort registered\nskill names deterministically before rendering, keeping membership, aliases,\nmetadata and text/JSON output formats unchanged, with tests for mixed-case\nnames and differing enumeration order.\n\nThis invocation covered ONLY the /plan-ceo-review mode decision, at the\nuser's request. Office Hours, onboarding/setup prompts, the implementation\napproach decision (0C-bis), and the 11-section deep review are deferred to a\nlater invocation.\n\n## System audit (light, for mode grounding)\n\n- Branch `edinburgh-v1`, clean tree, far ahead of `main` (633 files); no stash.\n- No per-branch design doc, no CEO handoff note.\n- Skill enumeration paths found in the repo:\n - `test/helpers/skill-census.ts:63` \u2014 `readdirSync` with NO sort; registry\n is a `Set`, so insertion order follows enumeration order.\n - `lib/context-bill.ts:382` \u2014 `findSkillDirs` returns `out.sort()`\n (code-point sort on full paths; mixed case orders `Z` before `a`).\n - `lib/context-bill.ts:461` \u2014 `diffBills` sorts merged skill names with the\n default comparator (same mixed-case caveat).\n - `scripts/skill-check.ts:101` \u2014 `readdirSync(hostDir).sort()`.\n- Related tests: `test/skill-census.test.ts`, `test/context-bill.test.ts`,\n `test/catalog-budget.test.ts`.\n\n## Review mode decision\n\n**Mode: HOLD SCOPE** (auto-decided from the user's saved preference on\n`plan-ceo-review-mode`; change with /plan-tune).\n\nWhy HOLD SCOPE: this is a small correctness fix / refactor to an existing\npath, with explicit invariants (membership, aliases, metadata, formats\nunchanged). The context-dependent default for bug fixes and refactors is\nHOLD SCOPE. The value is in rigor: a single shared comparator, mixed-case\nand enumeration-order tests, and no silent behavior change in text or JSON\noutput.\n\n## Open before the substantive review\n\n1. 0C-bis implementation approaches (minimal vs shared comparator helper)\n still need an explicit approval.\n2. Confirm which listing path the draft means (\"the existing listing path\"):\n the census helper, the context-bill CLI, or both.\n3. Deep review per `sections/review-sections.md`, then the\n `## GSTACK REVIEW REPORT` section.\n" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:24:26.395Z", + "toolUseId": "toolu_01EVSYPhAqVAdZhoeTZ9bU1r", + "kind": "result", + "isError": false + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "timestamp": "2026-09-10T04:32:13.463Z", + "toolUseId": "toolu_01TjWu1RBRGNXRWZ47Yro3ZT", + "kind": "use", + "name": "Skill", + "input": { + "skill": "plan-ceo-review" + } + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "timestamp": "2026-09-10T04:32:14.680Z", + "toolUseId": "toolu_01TjWu1RBRGNXRWZ47Yro3ZT", + "kind": "result", + "isError": false + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "timestamp": "2026-09-10T04:32:29.695Z", + "toolUseId": "toolu_01YWrQccueekQXWmbEqyiRfV", + "kind": "use", + "name": "Bash", + "input": { + "command": "_SS=\"$HOME/.claude/skills/gstack/bin/gstack-skill-start\"\n[ -x \"$_SS\" ] || _SS=\".claude/skills/gstack/bin/gstack-skill-start\"\n\"$_SS\" --skill \"plan-ceo-review\" --model \"claude\" --parent-pid \"$PPID\" \\\n || echo \"SKILL_START: unavailable \u2014 stale install; run ./setup or /gstack-upgrade (preamble degraded, continue the user's task)\"", + "description": "Run gstack skill-start preamble" + } + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "timestamp": "2026-09-10T04:32:31.475Z", + "toolUseId": "toolu_01YWrQccueekQXWmbEqyiRfV", + "kind": "result", + "isError": false + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "timestamp": "2026-09-10T04:32:48.387Z", + "toolUseId": "toolu_01G8mYVmSDWTrxTxuTBS8qbU", + "kind": "use", + "name": "Bash", + "input": { + "command": "git remote get-url origin 2>/dev/null; echo \"---PR---\"; gh pr view --json baseRefName -q .baseRefName 2>&1 | head -2; echo \"---DEFAULT---\"; gh repo view --json defaultBranchRef -q .defaultBranchRef.name 2>&1 | head -2; echo \"---SYMREF---\"; git symbolic-ref refs/remotes/origin/HEAD 2>/dev/null | sed 's|refs/remotes/origin/||'; git rev-parse --verify origin/main >/dev/null 2>&1 && echo \"origin/main exists\"", + "description": "Detect platform and base branch" + } + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "timestamp": "2026-09-10T04:32:49.956Z", + "toolUseId": "toolu_01Mn1fJWAVFxQKg4c9nDCTFz", + "kind": "use", + "name": "Bash", + "input": { + "command": "git log --oneline -8; echo \"---STASH---\"; git stash list | head -3; echo \"---DIFF vs main---\"; git diff main --stat 2>/dev/null | tail -3", + "description": "Quick system audit: log, stash, diff" + } + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "timestamp": "2026-09-10T04:32:50.104Z", + "toolUseId": "toolu_01G8mYVmSDWTrxTxuTBS8qbU", + "kind": "result", + "isError": false + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "timestamp": "2026-09-10T04:32:50.377Z", + "toolUseId": "toolu_01Mn1fJWAVFxQKg4c9nDCTFz", + "kind": "result", + "isError": false + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "timestamp": "2026-09-10T04:32:51.506Z", + "toolUseId": "toolu_01SA5VsDgqehzwJQMweeZNKi", + "kind": "use", + "name": "Bash", + "input": { + "command": "grep -n -i \"mode\\|ceo\" \"$HOME/.claude/skills/gstack/scripts/question-registry.ts\" 2>/dev/null | head -20", + "description": "Find question_id for CEO mode selection" + } + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "timestamp": "2026-09-10T04:32:51.542Z", + "toolUseId": "toolu_01SA5VsDgqehzwJQMweeZNKi", + "kind": "result", + "isError": false + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "timestamp": "2026-09-10T04:32:52.860Z", + "toolUseId": "toolu_01ATJDNJ62fCiTEdmPJ7qm4N", + "kind": "use", + "name": "Bash", + "input": { + "command": "grep -rn -i \"skill.*list\\|listSkills\\|list-skills\\|skills list\" --include=*.ts --include=*.sh -l bin setup lib scripts 2>/dev/null | head -15", + "description": "Locate the skill-listing code path the draft targets" + } + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "timestamp": "2026-09-10T04:32:52.908Z", + "toolUseId": "toolu_01ATJDNJ62fCiTEdmPJ7qm4N", + "kind": "result", + "isError": false + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "timestamp": "2026-09-10T04:33:01.877Z", + "toolUseId": "toolu_01Bwwgt4wwfrmyJDS7ZQMrpy", + "kind": "use", + "name": "Bash", + "input": { + "command": "printf '%s' \"Review mode for deterministic skill-list ordering draft: expansion, selective expansion, hold scope, or reduction\" | \"$HOME/.claude/skills/gstack/bin/gstack-question-preference\" --check \"plan-ceo-review-mode\" --summary-stdin 2>&1", + "description": "Check question-tuning preference for mode question" + } + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "timestamp": "2026-09-10T04:33:03.484Z", + "toolUseId": "toolu_01Bwwgt4wwfrmyJDS7ZQMrpy", + "kind": "result", + "isError": false + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "timestamp": "2026-09-10T04:33:28.691Z", + "toolUseId": "toolu_016K5grRiHEAyfUEUpX3dNxE", + "kind": "use", + "name": "Bash", + "input": { + "command": "\"$HOME/.claude/skills/gstack/bin/gstack-question-log\" '{\"skill\":\"plan-ceo-review\",\"question_id\":\"plan-ceo-review-mode\",\"question_summary\":\"Review mode for deterministic skill-list ordering draft\",\"category\":\"routing\",\"door_type\":\"two-way\",\"options_count\":4,\"user_choice\":\"C\",\"recommended\":\"C\",\"session_id\":\"656632-1789014751-94b2b37a\"}' 2>&1 || true", + "description": "Log auto-decided mode question" + } + }, + { + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "timestamp": "2026-09-10T04:33:30.094Z", + "toolUseId": "toolu_016K5grRiHEAyfUEUpX3dNxE", + "kind": "result", + "isError": false + } + ], + "options": { + "skillName": "plan-ceo-review", + "sessionId": "a836788b-1676-4ac3-a39d-bb86b873bf03", + "commandStartedAt": 1789014725874, + "now": 1789015327103 + } +} diff --git a/test/fixtures/auto-decide-saved-ai.json b/test/fixtures/auto-decide-saved-ai.json new file mode 100644 index 000000000..845c9b87d --- /dev/null +++ b/test/fixtures/auto-decide-saved-ai.json @@ -0,0 +1,291 @@ +{ + "transcript": { + "status": "ready", + "calls": [], + "assistantMessages": [ + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "text": "I'll invoke the `/plan-ceo-review` skill against the draft plan.", + "timestamp": "2026-09-10T04:21:58.556Z" + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "text": "Running the preamble first, then a light system audit so the mode question is grounded in the actual listing code.", + "timestamp": "2026-09-10T04:22:10.706Z" + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "text": "Found the relevant code paths. Running the question-tuning check for the mode question and handling the onboarding markers, then I'll present the mode decision.", + "timestamp": "2026-09-10T04:23:41.654Z" + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "text": "Heads-up from the preamble: unshipped work on this branch, so `/review` then `/ship` when you're ready. Also, gstack follows the **Boil the Ocean** principle: do the complete thing when AI makes the marginal cost near zero. Read more at https://garryslist.org/posts/boil-the-ocean if you'd like.\n\n**Review mode: HOLD SCOPE.**\n\nAuto-decided \"Select review mode\" \u2192 HOLD SCOPE (your saved preference on `plan-ceo-review-mode`). Change with /plan-tune.\n\nWhy this mode fits the draft:\n\n- **It is a correctness fix to an existing path, not a new capability.** Skill listings should not change order between machines or filesystems. The context default for bug fixes and refactors is HOLD SCOPE.\n- **The draft already states its invariants.** Membership, aliases, metadata and text/JSON formats stay unchanged. The review's job is to make those invariants bulletproof, not to widen them.\n- **The codebase confirms the problem is real.** The census helper enumerates skills with no sort at all. The context-bill code sorts with the default code-point comparator, so a mixed-case name like `Zeta` lands before `alpha`. Both are places the deep review should pin with tests.\n\nWhat I deliberately left open for the later invocation, at your request:\n\n1. **The implementation approach decision** (a shared case-insensitive comparator versus per-site sorts). The skill requires your explicit approval there before the deep review.\n2. **Which listing path the draft means.** The census helper, the context-bill CLI, or both. This changes the file list.\n3. **Onboarding prompts** (continuous checkpoint feature, cross-project learnings) and the Office Hours offer. Their markers are untouched, so they will fire again next healthy run.\n\nThe plan file at `.claude/plans/please-review-the-following-melodic-kernighan.md` records the mode, the audit findings with file and line references, and the three open items. I have not called ExitPlanMode since the review workflow is paused rather than complete.\n\nSTATUS: DONE. No durable learnings this session.", + "timestamp": "2026-09-10T04:24:41.750Z" + } + ] + }, + "tools": [ + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:22:00.757Z", + "toolUseId": "toolu_01D1z8pVbfkEeHZGgi6JdPZk", + "kind": "use", + "name": "Skill", + "input": { + "skill": "plan-ceo-review", + "args": "Draft: deterministic skill-list ordering. Users compare skill listings in scripts and reviews. Make the existing listing path sort registered skill names deterministically before rendering them. Keep skill membership, aliases, metadata and text/JSON output formats unchanged. Cover mixed-case names and differing directory enumeration order with tests. Review-mode decision only for this invocation." + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:22:02.528Z", + "toolUseId": "toolu_01D1z8pVbfkEeHZGgi6JdPZk", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:22:12.591Z", + "toolUseId": "toolu_01B89LH3gWatoKLbFjkW1duv", + "kind": "use", + "name": "Bash", + "input": { + "command": "_SS=\"$HOME/.claude/skills/gstack/bin/gstack-skill-start\"\n[ -x \"$_SS\" ] || _SS=\".claude/skills/gstack/bin/gstack-skill-start\"\n\"$_SS\" --skill \"plan-ceo-review\" --model \"claude\" --parent-pid \"$PPID\" \\\n || echo \"SKILL_START: unavailable \u2014 stale install; run ./setup or /gstack-upgrade (preamble degraded, continue the user's task)\"", + "description": "Run gstack skill-start preamble" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:22:15.395Z", + "toolUseId": "toolu_01B89LH3gWatoKLbFjkW1duv", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:22:42.272Z", + "toolUseId": "toolu_011L1hSC8vSZLDVGpfiokh1h", + "kind": "use", + "name": "Bash", + "input": { + "command": "git remote get-url origin 2>/dev/null; echo \"---LOG\"; git log --oneline -15; echo \"---STAT\"; git diff main --stat 2>/dev/null | tail -5; echo \"---STASH\"; git stash list | head; echo \"---DESIGN\"; SLUG=$(~/.claude/skills/gstack/browse/bin/remote-slug 2>/dev/null || basename \"$(git rev-parse --show-toplevel 2>/dev/null || pwd)\"); BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null | tr '/' '-'); echo \"SLUG=$SLUG BRANCH=$BRANCH\"; ls -t ~/.gstack/projects/$SLUG/*-design-*.md 2>/dev/null | head -1; ls -t docs/designs/*.md 2>/dev/null | head -3; ls -t ~/.gstack/projects/$SLUG/*-$BRANCH-ceo-handoff-*.md 2>/dev/null | head -1 || echo NO_HANDOFF; echo \"---LEARN\"; ~/.claude/skills/gstack/bin/gstack-config get cross_project_learnings 2>/dev/null || echo unset; ~/.claude/skills/gstack/bin/gstack-learnings-search --limit 5 2>/dev/null | head -20", + "description": "System audit: remote, log, diff, stash, design doc, learnings" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:22:43.881Z", + "toolUseId": "toolu_011L1hSC8vSZLDVGpfiokh1h", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:22:54.173Z", + "toolUseId": "toolu_01Jpe19C6Qy7iLjhxsVVFZEM", + "kind": "use", + "name": "Bash", + "input": { + "command": "cd /home/vercel-sandbox/gstack; ls bin | grep -i -E \"skill|list|catalog|census|route\" ; echo \"---GREP\"; grep -rl -i -E \"list.*skills|skills.*list|listSkills|skill-list|skills list\" --include=*.ts --include=*.sh --include=*.tmpl bin scripts lib _gstack-command setup 2>/dev/null | grep -v node_modules | head -30; echo \"---TODOS\"; grep -n -i -E \"sort|order|determin|listing\" TODOS.md | head -20", + "description": "Locate skill listing code and related TODOs" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:22:54.300Z", + "toolUseId": "toolu_01Jpe19C6Qy7iLjhxsVVFZEM", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:06.413Z", + "toolUseId": "toolu_013RE2TjYQMdqe8nLecHC4vw", + "kind": "use", + "name": "Bash", + "input": { + "command": "cd /home/vercel-sandbox/gstack; grep -n -E '\"(skill:check|skill:list|skills|list|catalog|context-bill)[^\"]*\":' package.json; echo \"---REGISTRY\"; grep -rn -i -E \"registered skill|registeredSkills|skill registry|SKILL_REGISTRY|listRegistered|registerSkill\" --include=*.ts --include=*.sh scripts lib bin test/helpers 2>/dev/null | grep -v node_modules | head -15; echo \"---READDIR\"; grep -rn -E \"readdirSync\\(\" scripts/gen-skill-docs.ts scripts/skill-check.ts scripts/skill-health.ts test/helpers/skill-census.ts lib/*.ts 2>/dev/null | head -20; echo \"---JSON\"; grep -rln -E \"\\-\\-json\" bin scripts --include=* 2>/dev/null | head -20", + "description": "Find skill enumeration and JSON-listing code" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:06.508Z", + "toolUseId": "toolu_013RE2TjYQMdqe8nLecHC4vw", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:13.375Z", + "toolUseId": "toolu_013Ee9LJJQ8R1fR2sQFPGwkj", + "kind": "use", + "name": "Read", + "input": { + "file_path": "/home/vercel-sandbox/gstack/test/helpers/skill-census.ts", + "offset": 40, + "limit": 50 + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:13.401Z", + "toolUseId": "toolu_013Ee9LJJQ8R1fR2sQFPGwkj", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:14.737Z", + "toolUseId": "toolu_01BU2xpwy4uMyx1Xp4quu9bj", + "kind": "use", + "name": "Read", + "input": { + "file_path": "/home/vercel-sandbox/gstack/lib/context-bill.ts", + "offset": 195, + "limit": 30 + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:14.759Z", + "toolUseId": "toolu_01BU2xpwy4uMyx1Xp4quu9bj", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:16.384Z", + "toolUseId": "toolu_013aumciUHw2Bqjaaq27g1EV", + "kind": "use", + "name": "Bash", + "input": { + "command": "cd /home/vercel-sandbox/gstack; grep -n -E \"json|text|format|\\.sort\\(|localeCompare\" lib/context-bill.ts | head -30; echo \"---BIN\"; grep -n -E \"json|sort|format\" bin/gstack-context-bill | head -15; echo \"---SKILLCHECK\"; grep -n -E \"json|\\.sort\\(\" scripts/skill-check.ts | head", + "description": "Check output formats and sorting in listing candidates" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:16.450Z", + "toolUseId": "toolu_013aumciUHw2Bqjaaq27g1EV", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:26.286Z", + "toolUseId": "toolu_015mJBMXZtpHKA6hWxDwTo8j", + "kind": "use", + "name": "Read", + "input": { + "file_path": "/home/vercel-sandbox/gstack/lib/context-bill.ts", + "offset": 335, + "limit": 50 + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:26.306Z", + "toolUseId": "toolu_015mJBMXZtpHKA6hWxDwTo8j", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:28.306Z", + "toolUseId": "toolu_01H7ZHzRW7U18u8zh3L6yKFH", + "kind": "use", + "name": "Bash", + "input": { + "command": "cd /home/vercel-sandbox/gstack; grep -n -E \"json|JSON|renderText|function render|function format\" bin/gstack-context-bill.ts bin/gstack-context-bill 2>/dev/null | head -20; ls bin | grep context-bill; echo \"---TESTS\"; ls test | grep -i -E \"context-bill|census|catalog\"; echo \"---GBRAIN\"; grep -n -E \"\\.skills\\b|skills:\" lib/context-bill.ts | head -10", + "description": "Find context-bill CLI output formats and tests" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:28.376Z", + "toolUseId": "toolu_01H7ZHzRW7U18u8zh3L6yKFH", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:45.519Z", + "toolUseId": "toolu_01BHQiS1mU8LZfxsWwFbGADs", + "kind": "use", + "name": "Bash", + "input": { + "command": "cd /home/vercel-sandbox/gstack; grep -n -i -E \"plan-ceo-review.*(mode|scope)|ceo.*mode\" ~/.claude/skills/gstack/scripts/question-registry.ts | head -5; echo \"---CHECK\"; printf '%s' \"Select review mode (expansion/selective/hold/reduction) for deterministic skill-list ordering plan\" | ~/.claude/skills/gstack/bin/gstack-question-preference --check \"plan-ceo-review-mode-selection\" --summary-stdin 2>&1 | head -5; echo \"---MARKERS\"; touch \"/tmp/gstack-paid-shard-pNXtSA/tmp/gstack-auto-decide-w2UU3e/.completeness-intro-seen\" && echo \"lake-intro marker touched\"", + "description": "Question-tuning check and lake-intro marker" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:47.061Z", + "toolUseId": "toolu_01BHQiS1mU8LZfxsWwFbGADs", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:53.288Z", + "toolUseId": "toolu_01Fz4MuCGi9tb8sPCq6SswEB", + "kind": "use", + "name": "Bash", + "input": { + "command": "cd /home/vercel-sandbox/gstack; printf '%s' \"Select review mode (expansion/selective/hold/reduction) for deterministic skill-list ordering plan\" | ~/.claude/skills/gstack/bin/gstack-question-preference --check \"plan-ceo-review-mode\" --summary-stdin 2>&1 | head -3", + "description": "Re-run tuning check with registered question id" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:23:54.770Z", + "toolUseId": "toolu_01Fz4MuCGi9tb8sPCq6SswEB", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:24:12.317Z", + "toolUseId": "toolu_01YHaTDadr9x4vFfuus4vamb", + "kind": "use", + "name": "Bash", + "input": { + "command": "cd /home/vercel-sandbox/gstack; ~/.claude/skills/gstack/bin/gstack-question-log '{\"skill\":\"plan-ceo-review\",\"question_id\":\"plan-ceo-review-mode\",\"question_summary\":\"Select review mode for deterministic skill-list ordering plan\",\"category\":\"routing\",\"door_type\":\"two-way\",\"options_count\":4,\"user_choice\":\"C\",\"recommended\":\"C\",\"session_id\":\"634003-1789014135-1db98f5a\"}' 2>/dev/null && echo \"question logged\" || echo \"question-log unavailable\"; ~/.claude/skills/gstack/bin/gstack-skill-end --skill \"plan-ceo-review\" --outcome success --session-id \"634003-1789014135-1db98f5a\" --tel-start \"1789014135\" --used-browse no --error-message \"\" --failed-step \"\" 2>/dev/null && echo \"telemetry ok\" || echo \"telemetry skipped\"", + "description": "Log auto-decided mode question and run skill-end telemetry" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:24:14.288Z", + "toolUseId": "toolu_01YHaTDadr9x4vFfuus4vamb", + "kind": "result", + "isError": false + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:24:26.317Z", + "toolUseId": "toolu_01EVSYPhAqVAdZhoeTZ9bU1r", + "kind": "use", + "name": "Write", + "input": { + "file_path": "/tmp/gstack-paid-shard-pNXtSA/tmp/gstack-hermetic-633854-8JWnVY/with-skills/.claude/plans/please-review-the-following-melodic-kernighan.md", + "content": "# Plan: deterministic skill-list ordering (CEO review \u2014 mode decision only)\n\n## Context\n\nUsers compare skill listings in scripts and reviews. Today the ordering of\nregistered skill names depends on directory enumeration order and, where a\nsort exists, on JavaScript's default code-point sort (uppercase before\nlowercase). The draft plan makes the existing listing path sort registered\nskill names deterministically before rendering, keeping membership, aliases,\nmetadata and text/JSON output formats unchanged, with tests for mixed-case\nnames and differing enumeration order.\n\nThis invocation covered ONLY the /plan-ceo-review mode decision, at the\nuser's request. Office Hours, onboarding/setup prompts, the implementation\napproach decision (0C-bis), and the 11-section deep review are deferred to a\nlater invocation.\n\n## System audit (light, for mode grounding)\n\n- Branch `edinburgh-v1`, clean tree, far ahead of `main` (633 files); no stash.\n- No per-branch design doc, no CEO handoff note.\n- Skill enumeration paths found in the repo:\n - `test/helpers/skill-census.ts:63` \u2014 `readdirSync` with NO sort; registry\n is a `Set`, so insertion order follows enumeration order.\n - `lib/context-bill.ts:382` \u2014 `findSkillDirs` returns `out.sort()`\n (code-point sort on full paths; mixed case orders `Z` before `a`).\n - `lib/context-bill.ts:461` \u2014 `diffBills` sorts merged skill names with the\n default comparator (same mixed-case caveat).\n - `scripts/skill-check.ts:101` \u2014 `readdirSync(hostDir).sort()`.\n- Related tests: `test/skill-census.test.ts`, `test/context-bill.test.ts`,\n `test/catalog-budget.test.ts`.\n\n## Review mode decision\n\n**Mode: HOLD SCOPE** (auto-decided from the user's saved preference on\n`plan-ceo-review-mode`; change with /plan-tune).\n\nWhy HOLD SCOPE: this is a small correctness fix / refactor to an existing\npath, with explicit invariants (membership, aliases, metadata, formats\nunchanged). The context-dependent default for bug fixes and refactors is\nHOLD SCOPE. The value is in rigor: a single shared comparator, mixed-case\nand enumeration-order tests, and no silent behavior change in text or JSON\noutput.\n\n## Open before the substantive review\n\n1. 0C-bis implementation approaches (minimal vs shared comparator helper)\n still need an explicit approval.\n2. Confirm which listing path the draft means (\"the existing listing path\"):\n the census helper, the context-bill CLI, or both.\n3. Deep review per `sections/review-sections.md`, then the\n `## GSTACK REVIEW REPORT` section.\n" + } + }, + { + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "timestamp": "2026-09-10T04:24:26.395Z", + "toolUseId": "toolu_01EVSYPhAqVAdZhoeTZ9bU1r", + "kind": "result", + "isError": false + } + ], + "options": { + "skillName": "plan-ceo-review", + "sessionId": "bd12c284-1989-47ed-9ffb-548b3d75a3ad", + "commandStartedAt": 1789014113697, + "now": 1789014714476 + } +} diff --git a/test/fixtures/autoplan-edit-prefix-ai.json b/test/fixtures/autoplan-edit-prefix-ai.json new file mode 100644 index 000000000..70ba6e554 --- /dev/null +++ b/test/fixtures/autoplan-edit-prefix-ai.json @@ -0,0 +1,53 @@ +{ + "sourceCommit": "12faead4636b97305348e25fc12258a56fcf6868", + "provenance": { + "path": ".context/ship-source-ai-delta-paid-20260910-v1/autoplan-edit-prefix-evidence-v1/public-tools.json", + "sha256": "e49a7b02986840f0e76466076d54b21d001bfc8ad94213258f24c19bdf968b91", + "projection": "Successful same-file use/result metadata plus exact current Edit request; unrelated events remain in context proof." + }, + "cwd": "/tmp/gstack-paid-shard-Am4ci8/tmp/gstack-autoplan-chain-9599im", + "ownedStateRoot": "/tmp/gstack-paid-shard-Am4ci8/tmp/gstack-hermetic-592891-MJaePo/skill-home-31EPP8/.gstack", + "viewportCapturedAt": "2026-09-10T04:27:38.073Z", + "viewport": " +r this plan. \n 80 +- Latency remedies, in order: if endpoint p95 > 300 ms for two consecutive stages, first add the per-me\n +mber predicate cache (a performance fix, exempt from the feature gate below); if still over, switch to \n +alternative B' (three parallel calls). The 600 ms guardrail is separate and triggers rollback, not reme\n +diation. \n 81 +- Gate: no new dashboard features (including the separately planned dark mode and personalization) unti\n +l the cohort metric has been read against control at the 25% stage. Performance remediation is exempt f\n +rom this gate. \n 82 +- Refresh semantics: SUCCESS and EMPTY panels enter REFRESHING (content stays visible); an ERROR panel \n +returns to LOADING on Refresh or Retry. \n 83 + \n 84 +## Spec review record \n 85 +Three adversarial review rounds (scores 7/10 \u2192 8/10 \u2192 8/10; 21 issues raised, 21 fixed in-document). Th\n +e loop hit its 3-iteration cap with the round-3 issues fixed inline above; no unresolved reviewer conce\n +rns remain. \n\n\u25cf Update(~/.gstack/projects/gstack-autoplan-chain-9599im/ceo-plans/2026-09-10-user-dashboard.md)\n\n\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n Edit file\n \u2026-592891-MJaePo/skill-home-31EPP8/.gstack/projects/gstack-autoplan-chain-9599im/ceo-plans/2026-09-10-user-dashboard.md\n\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\n 55 - Client analytics events: dashboard_viewed, dashboard_panel_state (ERROR/EMPTY only), dashboard_quick_action_click\n ed, dashboard_mark_all_read, dashboard_view_all_clicked, dashboard_refresh_clicked\n 56 \n 57 ## Deferred to TODOS.md\n 58 -All deferrals wait until the cohort metric has been read against control (see Gate below), then: \n 58 +Feature deferrals wait until the cohort metric has been read against control (see Gate below); the predicate cache \n +is a performance remedy and is exempt: \n 59 - Unread badge in global nav (P2)\n 60 - Smart-redirect experiment cohort for single-action members (P2)\n 61 - Real-time notification/activity updates via SSE (P3, infra decision)\n\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\u254c\n Do you want to make this edit to 2026-09-10-user-dashboard.md?\n \u276f 1. Yes\n 2. Yes, and switch to accept edits (auto-approve file edits and common file commands) for this session (shift+tab)\n 3. No\n\n Esc to cancel \u00b7 Tab to amend\n", + "before": "---\nstatus: ACTIVE\n---\n# CEO Plan: User Dashboard Page\nGenerated by /plan-ceo-review (via /autoplan) on 2026-09-10\nBranch: main | Mode: SELECTIVE EXPANSION\nRepo: gstack-autoplan-chain-9599im (fixture; no remote)\n\n## Baseline scope (unchanged from the source plan)\n- `/dashboard` page rendered after login, with three panels: QuickActions, NotificationsPanel, ActivityFeed\n- One aggregate endpoint `GET /api/dashboard` returning a top-level `fetchedAt` (server time of the response) plus one result per panel (`{ ok: true, data } | { ok: false, error }`), so one panel failing never blanks the page\n- \"Mark all as read\" behind a confirm dialog built on the existing dialog primitive\n- A toast primitive (new, built as a shared app-level component, success feedback only)\n- Canonical panel states, used everywhere in this record: LOADING (skeleton), EMPTY, ERROR, SUCCESS, REFRESHING (a SUCCESS or EMPTY panel re-fetching while its previous content stays visible). A shared `PanelFrame` component owns the chrome for all five; each panel supplies only its SUCCESS rendering.\n- Tailwind tokens; mobile-first sm/md/lg\n- Out of scope, from the source plan: dark mode, personalization (separate plans)\n\n## Vision\n\n### North star (12-month direction, NOT this release)\nThe dashboard that already knows what you were doing. The hero is the one eligible\n\"resume\" action, huge and one Tab away. Notifications are ranked by urgency and can be\ncleared per item. Activity streams in live. Nothing the member needs after login is\nmore than one click away, and nothing they don't need takes up space.\n\n### 10x within this release's constraints\nThe source plan forbids new mutation APIs and new infrastructure, so per-item read and\nlive streaming are deferred (E9 below, and the per-item read API in the deferred list).\nThe 10x lever available now is hierarchy: QuickActions first, Notifications second,\nActivity third, at every breakpoint. The per-panel result envelope and the shared\n`PanelFrame` are the widget contract that lets every later panel plug in without\nre-deciding its states.\n\n## Scope Decisions\n\n| # | Proposal | Effort | Decision | Reasoning |\n|---|----------|--------|----------|-----------|\n| E1 | \"Updated Xm ago\" + Refresh in page header | S | ACCEPTED | Uses the top-level `fetchedAt` of the aggregate response; Refresh puts panels into REFRESHING; makes staleness visible |\n| E2 | Notification rows deep-link to their target | S | ACCEPTED | Route targets exist per contracts; link only. The row stays unread after the member returns, until \"Mark all as read\" (intentional: no per-item read API in this release) |\n| E3 | `