import { expect, test } from 'bun:test'; import { ceoFirstReviewAUQ, ceoStep0Boundary, nativePlanCallFingerprint, planCountQuestionPhase } from './helpers/claude-pty-runner'; import captured from './fixtures/ceo-section-choice-ai.json'; import metadataCaptured from './fixtures/ceo-metadata-brief-ax.json'; import { E2E_TOUCHFILES } from './helpers/touchfiles-data'; function call(index = 4): any { const source = structuredClone(captured.calls[index]!); return { sessionId: source.sessionId, toolUseId: source.toolUseId, questions: source.questions, answered: true, failed: false, unansweredQuestionIndices: [], answeredAt: source.answeredAt, answers: Object.fromEntries(source.questions.map((q, i) => [q.question, source.answers[i]])) }; } const fp = (c: any) => nativePlanCallFingerprint(c, 0, true); function edit(c: any, change: (text: string) => string) { const q = c.questions[0], answer = c.answers[q.question]; q.question = change(q.question); c.answers = { [q.question]: answer }; } test('exact captured section choices start review; preceding actual setup does not', () => { let started = false; const classified: boolean[] = []; for (let i = 0; i < captured.calls.length; i++) { const question = fp(call(i)); expect(ceoFirstReviewAUQ(question)).toBe(captured.calls[i]!.expectedFirstReview); const phase = planCountQuestionPhase(question, started, ceoStep0Boundary, ceoFirstReviewAUQ); started = phase.reviewStarted; classified.push(phase.preReview); } expect(classified).toEqual([true, true, true, true, false, false, false, false, false]); }); test('an offered alternative and a quoted historical withdrawal retain current review identity', () => { const alternative = call(); alternative.answers[alternative.questions[0].question] = alternative.questions[0].options[1].label; expect(ceoFirstReviewAUQ(fp(alternative))).toBe(true); const quoted = call(); edit(quoted, s => s + '\nHistorical quote: "This issue has been resolved."'); expect(ceoFirstReviewAUQ(fp(quoted))).toBe(true); }); test.each([ ['pending', (c: any) => { c.answered = false; }], ['failed', (c: any) => { c.failed = true; }], ['unanswered index', (c: any) => { c.unansweredQuestionIndices = [0]; }], ['missing session', (c: any) => { c.sessionId = ''; }], ['missing tool id', (c: any) => { c.toolUseId = ''; }], ['unoffered answer', (c: any) => { c.answers[c.questions[0].question] = 'Not offered'; }], ['missing answer', (c: any) => { c.answers = {}; }], ['mixed packet', (c: any) => { c.questions.push(structuredClone(c.questions[0])); }], ['multi-select', (c: any) => { c.questions[0].multiSelect = true; }], ['duplicate options', (c: any) => { c.questions[0].options[1] = structuredClone(c.questions[0].options[0]); }], ['missing description', (c: any) => { c.questions[0].options[1].description = ''; }], ['option identity', (c: any) => { c.questions[0].options[1].label = '1B) Other'; }], ['section mismatch', (c: any) => edit(c, s => s.replace('Section 1 Architecture.', 'Section 2 Architecture.'))], ['recommendation mismatch', (c: any) => edit(c, s => s.replace('Recommendation: A', 'Recommendation: B'))], ['missing stakes', (c: any) => edit(c, s => s.replace(/^Stakes if we pick wrong:.*$/m, ''))], ['duplicate assessment', (c: any) => edit(c, s => s + '\nELI10: A second competing assessment.')], ['quoted assessment', (c: any) => edit(c, s => s.replace(/^ELI10: (.+)$/m, 'ELI10: "$1"'))], ['fenced context', (c: any) => edit(c, s => s.replace(/^(Project\/branch\/task:.*)$/m, '```\n$1\n```'))], ['Suppose assessment', (c: any) => edit(c, s => s.replace('ELI10: The plan', 'ELI10: Suppose the plan'))], ['single quoted assessment', (c: any) => edit(c, s => s.replace(/^ELI10: (.+)$/m, "ELI10: '$1'"))], ['current withdrawal', (c: any) => edit(c, s => s + '\nThis issue is withdrawn.')], ['completed withdrawal', (c: any) => edit(c, s => s + '\nWe have withdrawn this finding.')], ['conditional assessment', (c: any) => edit(c, s => s.replace('ELI10: The plan', 'ELI10: If the plan'))], ['withdrawn issue', (c: any) => edit(c, s => s + '\nWe withdraw this finding.')], ['resolved issue', (c: any) => edit(c, s => s + '\nThis issue has been resolved.')], ['administrative report', (c: any) => edit(c, s => s.replace(/^.*\n/, '1A — Should the completed review report be saved?\n'))], ['setup header', (c: any) => { c.questions[0].header = 'Setup'; }], ['borrowed qid', (c: any) => edit(c, s => s + '\n')], ])('rejects %s despite numbered review prose', (_name, mutate) => { const c = call(); mutate(c); expect(ceoFirstReviewAUQ(fp(c))).toBe(false); }); test('fingerprints cannot borrow another native call or its options', () => { const original = fp(call()); expect(ceoFirstReviewAUQ({ ...original, signature: 'other:tool' })).toBe(false); expect(ceoFirstReviewAUQ({ ...original, nativeCall: undefined })).toBe(false); expect(ceoFirstReviewAUQ({ ...original, options: original.options.slice(1) })).toBe(false); }); test('the regression inputs belong to the existing paid CEO case', () => { expect(E2E_TOUCHFILES['plan-ceo-finding-count']).toContain('test/ceo-section-choice-ai.test.ts'); expect(E2E_TOUCHFILES['plan-ceo-finding-count']).toContain('test/fixtures/ceo-section-choice-ai.json'); }); test('coherent finished-note destination is administrative, despite matching section and choice', () => { const c = call(), q = c.questions[0]; q.header = 'Destination'; q.question = '1A — Which storage location should hold these notes?\nProject/branch/task: main, Stripe payment webhook plan, Section 1 Architecture.\nELI10: The review is finished; these notes can be saved in either folder for convenience.\nStakes if we pick wrong: People may have to look in a second folder.\nRecommendation: A because the existing folder is easier to find.'; q.options = [{label:'A) Save beside the plan',description:'Keeps the finished notes together.'},{label:'B) Save in another folder',description:'Keeps finished notes separate.'}]; c.answers = {[q.question]: q.options[0].label}; expect(ceoFirstReviewAUQ(fp(c))).toBe(false); }); test('conditional stakes remain valid when the assessment asserts the current gap', () => { const c = call(); edit(c, s => s.replace('Stakes if we pick wrong:', 'Stakes if we pick wrong: Suppose there were an issue.')); expect(ceoFirstReviewAUQ(fp(c))).toBe(true); }); test('negated gaps and administrative missing fields cannot borrow review identity', () => { const negated = call(); edit(negated, s => s.replace(/^ELI10: .+$/m, 'ELI10: The transaction order is not unspecified. The plan guarantees commit before email.')); expect(ceoFirstReviewAUQ(fp(negated))).toBe(false); const admin = call(), q = admin.questions[0]; q.header = 'Destination'; q.question = '1A — Which storage location should hold these notes?\nProject/branch/task: main, Stripe payment webhook plan, Section 1 Architecture.\nELI10: These finished notes have a missing storage location.\nStakes if we pick wrong: People may look in the wrong folder.\nRecommendation: A because a notes folder is easy to find.'; q.options = [{label:'A) Add a notes folder',description:'Save the finished notes together.'},{label:'B) Use the existing folder',description:'No new folder.'}]; admin.answers = {[q.question]: q.options[0].label}; expect(ceoFirstReviewAUQ(fp(admin))).toBe(false); }); function metadataCall(): any { const c = structuredClone(metadataCaptured.call); return { ...c, answered: true, failed: false, unansweredQuestionIndices: [], answers: { [c.questions[0]!.question]: metadataCaptured.answer } }; } test('AX ordinary D-number question keeps its exact completed review identity', () => { const c = metadataCall(), before = JSON.stringify(c); expect(ceoFirstReviewAUQ(fp(c))).toBe(true); expect(planCountQuestionPhase(fp(c), false, ceoStep0Boundary, ceoFirstReviewAUQ)).toMatchObject({ preReview: false, reviewStarted: true }); expect(JSON.stringify(c)).toBe(before); expect(E2E_TOUCHFILES['plan-ceo-finding-count']).toContain('test/fixtures/ceo-metadata-brief-ax.json'); }); test('decision counter, review name and an alternative selection do not dictate the finding', () => { const c = metadataCall(); edit(c, s => s.replace(/^D5 /, 'D17 ').replace('Section 2 (Error & Rescue Map)', 'Section 3 (Failure Handling)')); c.answers[c.questions[0].question] = c.questions[0].options[1].label; expect(ceoFirstReviewAUQ(fp(c))).toBe(true); edit(c, s => s + '\nHistorical quote: "This finding is withdrawn."'); expect(ceoFirstReviewAUQ(fp(c))).toBe(true); }); test('metadata cannot replace native completion, current context or a real defect', () => { for (const mutate of [ (c: any) => { c.answered = false; }, (c: any) => { c.failed = true; }, (c: any) => { c.answeredAt = 'not a date'; }, (c: any) => { c.unansweredQuestionIndices = [0]; }, (c: any) => { c.answers = { other: metadataCaptured.answer }; }, (c: any) => { c.questions[0].header = 'Setup'; }, (c: any) => { c.questions[0].header = 'Section 9'; }, (c: any) => edit(c, s => s.replace('Section 2 (Error & Rescue Map)', 'Section 2 (Error & Rescue Map), Section 3 (Security)')), (c: any) => edit(c, s => s.replace('of the CEO review', 'of an earlier CEO review')), (c: any) => edit(c, s => s.replace(/^Project\/branch\/task: (.*)$/m, 'Project/branch/task: If approved, $1')), (c: any) => edit(c, s => s.replace(/^Project\/branch\/task:.*\n/m, '')), (c: any) => edit(c, s => s.replace(/^ELI10: (.*)$/m, 'ELI10: "$1"')), (c: any) => edit(c, s => s.replace(/^ELI10: .+$/m, 'ELI10: The handler has no current defect and needs no amendment.')), (c: any) => edit(c, s => s.replace(/^ELI10: .+$/m, 'ELI10: The handler commits before mail and already rescues every required error.')), (c: any) => edit(c, s => s.replace('ELI10: After', 'ELI10: Hypothetical example: after')), (c: any) => edit(c, s => s + '\nThis finding is withdrawn.'), (c: any) => edit(c, s => s + '; This finding is `no longer current`.'), (c: any) => edit(c, s => s + '\nThis finding is unproven.'), ]) { const c = metadataCall(); mutate(c); expect(ceoFirstReviewAUQ(fp(c))).toBe(false); } expect(ceoFirstReviewAUQ({ ...fp(metadataCall()), signature: 'foreign:call' })).toBe(false); }); test('a missing or withdrawn offered remedy cannot borrow metadata or an old assessment', () => { for (const mutate of [ (c: any) => { c.questions[0].options = [{ label: 'A: Keep the current handler', description: 'No code change.' }, { label: 'B: Save the review notes', description: 'Archive the current report.' }]; c.answers = { [c.questions[0].question]: c.questions[0].options[0].label }; }, (c: any) => { c.questions[0].options[0].description += '; This option is `withdrawn`.'; c.questions[0].options[2].description += '\nThis option is withdrawn.'; }, (c: any) => { c.questions[0].options.forEach((o: any) => { o.description = 'Hypothetical example. ' + o.description; }); }, ]) { const c = metadataCall(); mutate(c); expect(ceoFirstReviewAUQ(fp(c))).toBe(false); } }); function metadataRetryCall(): any { const c = structuredClone(metadataCaptured.retry.call); return { ...c, answered: true, failed: false, unansweredQuestionIndices: [], answers: { [c.questions[0]!.question]: metadataCaptured.retry.answer } }; } test('the separately failed AX retry binds its Issue annotation, bare choices and named plan', () => { const c = metadataRetryCall(), before = JSON.stringify(c); expect(ceoFirstReviewAUQ(fp(c))).toBe(true); expect(planCountQuestionPhase(fp(c), false, ceoStep0Boundary, ceoFirstReviewAUQ)) .toMatchObject({ preReview: false, reviewStarted: true }); expect(JSON.stringify(c)).toBe(before); c.answers[c.questions[0].question] = c.questions[0].options[2].label; expect(ceoFirstReviewAUQ(fp(c))).toBe(true); }); test('a reviewed filename and section identity can be consistently renamed', () => { const c = metadataRetryCall(); edit(c, s => s.replace(/^D4 /, 'D12 ').replace(/Issue 2\.1/, 'Issue 8.3') .replace('Section 2 (Error & Rescue Map)', 'Section 8 (Notification Handling)') .replace(/PLAN\.md/g, 'plans/checkout-flow.md')); expect(ceoFirstReviewAUQ(fp(c))).toBe(true); edit(c, s => s + '\nHistorical quote: "This issue is withdrawn."'); expect(ceoFirstReviewAUQ(fp(c))).toBe(true); edit(c, s => s.replace(/\s*]+>/, '')); expect(ceoFirstReviewAUQ(fp(c))).toBe(true); }); test('retry metadata cannot borrow a foreign section, plan, source or incomplete native call', () => { for (const [index, mutate] of [ (c: any) => { c.answered = false; }, (c: any) => { c.failed = true; }, (c: any) => { c.answeredAt = 'unknown'; }, (c: any) => { c.unansweredQuestionIndices = [0]; }, (c: any) => { c.questions[0].header = 'Issue 8.1'; }, (c: any) => edit(c, s => s.replace('Issue 2.1', 'Issue 3.1')), (c: any) => edit(c, s => s.replace('Section 2 (Error & Rescue Map)', 'Section 3 (Security)')), (c: any) => edit(c, s => s.replace('CEO review of PLAN.md,', 'CEO review of DIFFERENT.md,')), (c: any) => edit(c, s => s.replace("PLAN.md says 'no error handling on the email leg'", "OTHER.md says 'no error handling on the email leg'")), (c: any) => edit(c, s => s.replace('CEO review of PLAN.md,', 'Historical CEO review of PLAN.md,')), (c: any) => edit(c, s => s.replace(/^Project\/branch\/task: (.+)$/m, 'Project/branch/task: If approved, $1')), (c: any) => edit(c, s => s.replace(/^ELI10: (.+)$/m, 'ELI10: "$1"')), (c: any) => edit(c, s => s.replace('ELI10: The handler', 'ELI10: Source excerpt: the handler')), (c: any) => edit(c, s => s.replace('PLAN.md says', 'If approved, PLAN.md says')), (c: any) => edit(c, s => s.replace('PLAN.md says', 'PLAN.md does not say')), (c: any) => edit(c, s => s.replace('plan-ceo-review-mail-rescue', 'plan-ceo-review-setup')), (c: any) => edit(c, s => s + '\n'), ].entries()) { const c = metadataRetryCall(); mutate(c); expect(ceoFirstReviewAUQ(fp(c)), `retry mutation ${index}`).toBe(false); } }); test('current withdrawal and a withdrawn offered amendment override the retry brief', () => { for (const change of [ (s: string) => s + '\nThis issue is withdrawn.', (s: string) => s + '; This finding is `no longer current`.', (s: string) => s + '\nIssue 2.1 is withdrawn.', (s: string) => s.replace(/^ELI10: .+$/m, 'ELI10: The handler has no current defect and needs no amendment.'), ]) { const c = metadataRetryCall(); edit(c, change); expect(ceoFirstReviewAUQ(fp(c))).toBe(false); } const c = metadataRetryCall(); c.questions[0].options[0].description += '; This option is `withdrawn`.'; expect(ceoFirstReviewAUQ(fp(c))).toBe(false); });