import { describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { pathToFileURL } from 'node:url'; import { autoplanPhaseCompletions } from './helpers/autoplan-phase-observer'; import type { PlanCountTranscript } from './helpers/plan-count-transcript'; import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles'; const START = Date.parse('2026-09-08T16:00:00.000Z'); const transcript = (...messages: Array<[number, string]>): PlanCountTranscript => ({ status: 'ready', calls: [], assistantMessages: messages.map(([ms, text]) => ({ sessionId: 'autoplan-fixture', timestamp: new Date(START + ms).toISOString(), text, })), }); describe('native autoplan phase observation', () => { test('AF public wrapped-up announcement retains its actual Phase 1 timestamp', () => { // Exact reader-projected public parent narration from the AF run; no raw // signature or private block is stored. The next-phase mention adds no hit. const announcement = { sessionId: '6ce27a04-7422-4687-9de9-138e68f308d8', text: 'Phase 1 wrapped up: 11 findings from the Claude subagent, 30 of 34 spec issues fixed after 3 review rounds, and 21 obligations carried forward with 3 disagreements flagged as taste items. Moving on to Phase 2 (design review) now that UI scope was detected.\n\n', timestamp: '2026-09-10T00:00:03.893Z', }; const at = Date.parse(announcement.timestamp); expect(autoplanPhaseCompletions({ status: 'ready', calls: [], assistantMessages: [announcement] }, at - 1)).toEqual([{ phase: 1, ts: at }]); }); test('AF bare done announcement retains Phase 2 without crediting its DX transition', () => { const announcement2 = { sessionId: "6ce27a04-7422-4687-9de9-138e68f308d8", text: "Phase 2 done: Claude subagent found 15 issues (1 critical, 8 high, 6 medium), with 14 fully accepted and 1 partially accepted; design score rose from 6/10 to 8.7/10, and 10 accepted items are now carried into the Implementation plan. Moving on to Phase 2.5 (DX Review) since developer-facing scope was detected.", timestamp: "2026-09-10T00:08:31.892Z", }; const at = Date.parse(announcement2.timestamp); expect(autoplanPhaseCompletions({ status: 'ready', calls: [], assistantMessages: [announcement2] }, at - 1)).toEqual([{ phase: 2, ts: at }]); }); test('AF named DX completion retains Phase 2.5 without crediting its Eng transition', () => { const announcement3 = { sessionId: "6ce27a04-7422-4687-9de9-138e68f308d8", text: "Phase 2.5 (DX review) is done: DX score rose from 5.1 to 8.0/10, all 19 findings reviewed with 12 items accepted into the plan and one taste item flagged for gating. All pre-checks pass, so I'm moving on to Phase 3, the final Engineering Review of the amended plan.\n\n", timestamp: "2026-09-10T00:16:29.888Z", }; const at = Date.parse(announcement3.timestamp); expect(autoplanPhaseCompletions({ status: 'ready', calls: [], assistantMessages: [announcement3] }, at - 1)).toEqual([{ phase: 2.5, ts: at }]); }); test('completed-state declarations retain supported phases, punctuation and first native time', () => { for (const state of ['wrapped up', 'done']) for (const phase of [1, 2, 2.5, 3]) for (const tail of ['', '.', ': Work retained.', '. Work retained.']) { expect(autoplanPhaseCompletions(transcript([1, `Phase ${phase} ${state}${tail}`]), START)) .toEqual([{ phase, ts: START + 1 }]); } expect(autoplanPhaseCompletions(transcript([1, '**Phase 1 wrapped up.**']), START)) .toEqual([{ phase: 1, ts: START + 1 }]); expect(autoplanPhaseCompletions(transcript([4, 'Phase 1 wrapped up.'], [2, 'Phase 3 wrapped up.'], [3, 'Phase 1 wrapped up: Moving to Phase 2.']), START)) .toEqual([{ phase: 3, ts: START + 2 }, { phase: 1, ts: START + 3 }]); }); test('affirmative completion words share punctuation and optional is without future tense', () => { for (const state of ['complete', 'completed', 'done', 'finished', 'wrapped up']) { for (const copula of ['', 'is ']) for (const tail of ['', '.', ': Work retained.']) { expect(autoplanPhaseCompletions(transcript([1, `Phase 3 ${copula}${state}${tail}`]), START)) .toEqual([{ phase: 3, ts: START + 1 }]); } for (const text of [`Phase 3 ${state}?`, `Phase 3 will be ${state}.`, `Phase 3 is not ${state}.`, `Phase 3 ${state} if the reviewer finishes.`, `Phase 3 ${state} when the work ends.`, `**Phase 3 ${state}** if approved.`, `Example:\nPhase 3 ${state}.`, `> Phase 3 ${state}.`, `Phase 3 (Eng review) is not ${state}.`, `Phase 3 (Eng review) ${state} if approved.`]) { expect(autoplanPhaseCompletions(transcript([1, text]), START), text).toEqual([]); } } }); test('known phase names must agree with their number and never supply completion alone', () => { const names = [[1, 'CEO'], [2, 'Design'], [2.5, 'DX'], [3, 'Eng'], [3, 'Engineering']] as const; for (const [phase, name] of names) for (const suffix of ['', ' review']) { const declaration = `Phase ${phase} (${name}${suffix}) is finished.`; expect(autoplanPhaseCompletions(transcript([1, declaration]), START)).toEqual([{ phase, ts: START + 1 }]); expect(autoplanPhaseCompletions(transcript([1, `**${declaration}**`]), START)).toEqual([{ phase, ts: START + 1 }]); for (const other of [1, 2, 2.5, 3].filter(n => n !== phase)) { expect(autoplanPhaseCompletions(transcript([1, declaration.replace(`Phase ${phase}`, `Phase ${other}`)]), START)) .toEqual([]); } } for (const text of ['Phase 3 (Eng review).', 'Phase 2.5 (future DX review) is done.', 'Phase 2.5 (DX review if approved) is done.', 'Phase 1 (source) complete.', 'Phase 2.5 ((DX review)) is done.', 'Phase 2.5 (DX review) finished soon.', '# Phase 2.5 (DX review) is done.', 'Example:\nPhase 2.5 (DX review) is done.']) { expect(autoplanPhaseCompletions(transcript([1, text]), START), text).toEqual([]); } }); test('future, conditional, negative and quoted wrap-up claims do not complete a phase', () => { for (const text of [ 'Phase 1 will wrap up.', 'Phase 1 has not wrapped up.', 'Phase 1 is not wrapped up.', 'Phase 1 wrapped up if the reviewer finishes.', 'Phase 1 wrapped up when the review ends.', 'Phase 1 wrapped up but is not complete.', 'Phase 1 wrapped up?', 'Once Phase 1 wrapped up, we would start Phase 2.', 'I will announce Phase 1 wrapped up.', '**Phase 1 wrapped up** if the tests pass.', 'Phase 4 wrapped up.', 'Phase 2.1 wrapped up.', '# Phase 1 wrapped up.', '> Phase 1 wrapped up.', '"Phase 1 wrapped up."', '- Phase 1 wrapped up.', '| Phase 1 wrapped up. |', ' Phase 1 wrapped up.', '```text\nPhase 1 wrapped up.\n```', '~~~text\nPhase 1 wrapped up.\n~~~', 'Example:\nPhase 1 wrapped up.\nPhase 2 wrapped up.', 'The template says:\n\nPhase 1 wrapped up.', '**Phase 1 wrapped up.** Emit phase-transition summary:', ]) for (const declaration of [text, text.replace(/wrapped up/g, 'done')]) { expect(autoplanPhaseCompletions(transcript([1, declaration]), START), declaration).toEqual([]); } }); test('wrapped-up declarations retain ready transcript and native timestamp requirements', () => { const current = transcript([1, 'Phase 1 wrapped up.']); for (const status of ['missing', 'error'] as const) { expect(autoplanPhaseCompletions({ ...current, status }, START)).toEqual([]); } expect(autoplanPhaseCompletions(current, START + 2)).toEqual([]); expect(autoplanPhaseCompletions({ ...current, assistantMessages: current.assistantMessages.map( message => ({ ...message, timestamp: 'invalid' })) }, START)).toEqual([]); }); test('retains actual completion timestamps when several phases arrive between polls', () => { expect(autoplanPhaseCompletions(transcript( [1, '**Phase 1 complete.** Codex: 2 concerns. Native: 3 issues.'], [2, 'Phase 2 complete. Design outputs are in the plan.'], [3, '**Phase 2.5 complete.** DX overall: 8/10.'], [4, 'Phase 3 complete. Both engineering reviews finished.'], ), START)).toEqual([1, 2, 2.5, 3].map((phase, index) => ({ phase, ts: START + index + 1 }))); }); test('duplicate announcements retain their first timestamp without reordering phases', () => { expect(autoplanPhaseCompletions(transcript( [3, 'Phase 1 complete.'], [2, 'Phase 2 complete.'], [1, '**Phase 1 complete.**'], [4, '**Phase 3 complete.**'], ), START)).toEqual([{ phase: 1, ts: START + 1 }, { phase: 2, ts: START + 2 }, { phase: 3, ts: START + 4 }]); }); test('headings, quoted skill text, examples, tables and planned checklists provide no completion', () => { for (const text of [ '## Phase 3 complete.', '**PHASE 3 COMPLETE.** Emit phase-transition summary:', '> **Phase 3 complete.** Codex: [N concerns].', 'The section says "**Phase 3 complete.**".', 'Example: **Phase 3 complete.**', 'Example announcement:\n**Phase 3 complete.**', 'Example announcement:\nPhase 2 complete.\nPhase 3 complete.', 'The template requires this completion marker:\n\n**Phase 3 complete.**', ' **Phase 3 complete.**', '\tPhase 3 complete.', '```markdown\n**Phase 3 complete.**\n```', '```markdown\n```still-code\nPhase 3 complete.\n```', '````markdown\n```\nPhase 3 complete.\n````', '~~~markdown\nPhase 3 complete.\n~~~', '```markdown\nPhase 3 complete.', '| **Phase 3 complete.** | pending |', '- [ ] **Phase 3 complete.**', '1. Phase 3 complete.', 'I will announce **Phase 3 complete.** after the review.', 'Phase 3 complete when the engineering review ends.', 'Phase 3 complete?', 'Phase 4 complete.', ]) expect(autoplanPhaseCompletions(transcript([1, text]), START), text).toEqual([]); }); test('a real announcement after a closed example fence still establishes completion', () => { expect(autoplanPhaseCompletions(transcript([1, 'Example:\n```markdown\nPhase 3 complete.\n```\n\n**Phase 1 complete.**']), START)) .toEqual([{ phase: 1, ts: START + 1 }]); }); test('accepts a template-compliant quoted transition with actual consensus while rejecting quoted examples', () => { const actual = '> **Phase 1 complete.** Codex: 2 concerns. Claude subagent: 3 issues.\n' + '> Consensus: 4/6 confirmed, 2 disagreements → surfaced at gate.\n> Passing to Phase 2.'; expect(autoplanPhaseCompletions(transcript([1, actual]), START)).toEqual([{ phase: 1, ts: START + 1 }]); expect(autoplanPhaseCompletions(transcript([1, actual.replace('Codex: 2 concerns.', 'Codex: unavailable.')]), START)) .toEqual([{ phase: 1, ts: START + 1 }]); for (const quoted of [ actual.replace('4/6', 'X/6'), actual.replace('2 concerns', '[N concerns]'), 'The template contains this example:\n' + actual, 'Example announcement:\n' + actual + '\n' + actual.replace('Phase 1', 'Phase 3'), '> **Phase 3 complete.**', ]) expect(autoplanPhaseCompletions(transcript([1, quoted]), START), quoted).toEqual([]); }); test('missing/error transcripts and pre-command declarations cannot establish coverage', () => { const prior = transcript([-1, '**Phase 3 complete.**']); expect(autoplanPhaseCompletions(prior, START)).toEqual([]); for (const status of ['missing', 'error'] as const) { expect(autoplanPhaseCompletions({ ...transcript([1, '**Phase 3 complete.**']), status }, START)).toEqual([]); } }); test('does not repair an out-of-order chain or synthesize omitted phases', () => { expect(autoplanPhaseCompletions(transcript([1, 'Phase 3 complete.'], [2, 'Phase 1 complete.']), START)) .toEqual([{ phase: 3, ts: START + 1 }, { phase: 1, ts: START + 2 }]); expect(autoplanPhaseCompletions(transcript([1, 'Phase 2 skipped — no UI scope.']), START)).toEqual([]); }); test('phase observer changes select the autoplan eval', () => { for (const file of ['test/helpers/autoplan-phase-observer.ts', 'test/autoplan-phase-observer.test.ts']) { expect(selectTests([file], E2E_TOUCHFILES).selected).toEqual(['autoplan-chain-pty']); } }); test.skipIf(process.platform === 'win32')('ANSI-rendered completions use native evidence while displayed Read/source markers do not', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'autoplan-phase-replay-')); const fake = path.join(dir, 'fake-claude'); const worker = path.join(dir, 'worker.ts'); const recordFile = path.join(dir, 'events.jsonl'); const resultFile = path.join(dir, 'result.json'); fs.writeFileSync(fake, `#!${process.execPath}\n` + String.raw` import * as fs from 'node:fs'; import * as path from 'node:path'; fs.writeFileSync(process.env.PHASE_RECORD, JSON.stringify({pid:process.pid}) + '\n'); const sessionId = 'fake-autoplan-session'; const dir = path.join(process.env.CLAUDE_CONFIG_DIR, 'projects', 'fixture'); fs.mkdirSync(dir, {recursive:true}); const base = Date.now(); const write = (role, content, offset, extra = {}) => fs.appendFileSync(path.join(dir, sessionId + '.jsonl'), JSON.stringify({ sessionId, cwd:process.cwd(), isSidechain:false, timestamp:new Date(base + offset).toISOString(), message:{role,content}, ...extra, }) + '\n'); write('user', [{type:'tool_result', tool_use_id:'read', content:'**Phase 3 complete.**'}], 0); write('assistant', [{type:'text', text:'> **Phase 3 complete.** is the quoted source marker.'}], 0); write('assistant', [{type:'text', text:'**Phase 3 wrapped up.**'}], 0, {isSidechain:true}); write('assistant', [{type:'text', text:'**Phase 3 wrapped up.**'}], 0, {cwd:path.join(process.cwd(), 'foreign')}); process.stdin.setRawMode?.(true); let sent = false; process.stdin.on('data', data => { if (sent || !data.toString().includes('\r')) return; sent = true; process.stdout.write('\x1b[2J\x1b[H'); [1, 2, 2.5, 3].forEach((phase, index) => { const message = '**Phase ' + phase + (phase === 1 ? ' wrapped up.**' : phase === 2 ? ' done.**' : phase === 2.5 ? ' (DX review) is finished.**' : ' complete.**'); write('assistant', [{type:'text', text:message}], index + 1); process.stdout.write('● \x1b[1mPhase ' + phase + ' complete.\x1b[22m\n'); }); process.stdout.write('NATIVE_PHASES_READY\n'); }); process.stdout.write('Read: autoplan/sections/eng-phase.md\n> **Phase 3 complete.**\nSOURCE_READY\n'); process.on('SIGINT', () => process.exit(0)); process.stdin.resume(); `); fs.chmodSync(fake, 0o755); const moduleUrl = (name: string) => pathToFileURL(path.resolve(import.meta.dir, 'helpers', name)).href; fs.writeFileSync(worker, ` import { launchClaudePty } from ${JSON.stringify(moduleUrl('claude-pty-runner.ts'))}; import { readPlanCountTranscript } from ${JSON.stringify(moduleUrl('plan-count-transcript.ts'))}; import { autoplanPhaseCompletions } from ${JSON.stringify(moduleUrl('autoplan-phase-observer.ts'))}; const start = Date.now(); const session = await launchClaudePty({cwd:${JSON.stringify(dir)}, timeoutMs:8000, env:{PHASE_RECORD:${JSON.stringify(recordFile)}}}); try { await session.waitFor('SOURCE_READY', {timeoutMs:4000, pollMs:20}); const read = () => readPlanCountTranscript(session.hermeticConfigDir, ${JSON.stringify(dir)}); const sourceOnly = autoplanPhaseCompletions(read(), start); const oldPattern = /\\*\\*Phase\\s+(\\d+(?:\\.\\d+)?)\\s+complete\\.?\\*\\*/g; const sourceFalsePositive = [...session.visibleText().matchAll(oldPattern)].length; const since = session.mark(); session.send('\\r'); await session.waitFor('NATIVE_PHASES_READY', {timeoutMs:4000, pollMs:20}); const rendered = session.visibleSince(since); const oldPatternMissed = [...rendered.matchAll(oldPattern)].length; const hits = autoplanPhaseCompletions(read(), start); await Bun.write(${JSON.stringify(resultFile)}, JSON.stringify({sourceOnly, sourceFalsePositive, oldPatternMissed, hits, rendered})); } finally { await session.close(); } `); const child = Bun.spawn([process.execPath, worker], { env: { ...process.env, BROWSE_TERMINAL_BINARY: fake, EVALS_HERMETIC: '1' }, stdout: 'pipe', stderr: 'pipe', }); const timer = setTimeout(() => child.kill('SIGKILL'), 12_000); try { const [code, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]); expect(code, stdout + stderr).toBe(0); const result = JSON.parse(fs.readFileSync(resultFile, 'utf8')); expect(result.sourceOnly).toEqual([]); expect(result.sourceFalsePositive).toBe(1); expect(result.oldPatternMissed).toBe(0); expect(result.rendered).toContain('Phase 3 complete.'); expect(result.hits.map((hit: { phase: number }) => hit.phase)).toEqual([1, 2, 2.5, 3]); for (let index = 1; index < result.hits.length; index++) { expect(result.hits[index].ts).toBeGreaterThan(result.hits[index - 1].ts); } const pid = JSON.parse(fs.readFileSync(recordFile, 'utf8').split('\n')[0]!).pid; expect(() => process.kill(pid, 0)).toThrow(); } finally { clearTimeout(timer); child.kill('SIGKILL'); if (fs.existsSync(recordFile)) { const pid = JSON.parse(fs.readFileSync(recordFile, 'utf8').split('\n')[0]!).pid; try { process.kill(pid, 'SIGKILL'); } catch { /* already reaped */ } } fs.rmSync(dir, {recursive:true, force:true}); } }, 15_000); });