Files
gstack/test/autoplan-phase-observer.test.ts
T

301 lines
17 KiB
TypeScript

import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { pathToFileURL } from 'node:url';
import { autoplanPhaseCompletions } from './helpers/autoplan-phase-observer';
import type { PlanCountTranscript } from './helpers/plan-count-transcript';
import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles';
const START = Date.parse('2026-09-08T16:00:00.000Z');
const transcript = (...messages: Array<[number, string]>): PlanCountTranscript => ({
status: 'ready', calls: [],
assistantMessages: messages.map(([ms, text]) => ({
sessionId: 'autoplan-fixture', timestamp: new Date(START + ms).toISOString(), text,
})),
});
describe('native autoplan phase observation', () => {
test('AF public wrapped-up announcement retains its actual Phase 1 timestamp', () => {
// Exact reader-projected public parent narration from the AF run; no raw
// signature or private block is stored. The next-phase mention adds no hit.
const announcement = {
sessionId: '6ce27a04-7422-4687-9de9-138e68f308d8',
text: 'Phase 1 wrapped up: 11 findings from the Claude subagent, 30 of 34 spec issues fixed after 3 review rounds, and 21 obligations carried forward with 3 disagreements flagged as taste items. Moving on to Phase 2 (design review) now that UI scope was detected.\n\n',
timestamp: '2026-09-10T00:00:03.893Z',
};
const at = Date.parse(announcement.timestamp);
expect(autoplanPhaseCompletions({ status: 'ready', calls: [],
assistantMessages: [announcement] }, at - 1)).toEqual([{ phase: 1, ts: at }]);
});
test('AF bare done announcement retains Phase 2 without crediting its DX transition', () => {
const announcement2 = {
sessionId: "6ce27a04-7422-4687-9de9-138e68f308d8",
text: "Phase 2 done: Claude subagent found 15 issues (1 critical, 8 high, 6 medium), with 14 fully accepted and 1 partially accepted; design score rose from 6/10 to 8.7/10, and 10 accepted items are now carried into the Implementation plan. Moving on to Phase 2.5 (DX Review) since developer-facing scope was detected.",
timestamp: "2026-09-10T00:08:31.892Z",
};
const at = Date.parse(announcement2.timestamp);
expect(autoplanPhaseCompletions({ status: 'ready', calls: [],
assistantMessages: [announcement2] }, at - 1)).toEqual([{ phase: 2, ts: at }]);
});
test('AF named DX completion retains Phase 2.5 without crediting its Eng transition', () => {
const announcement3 = {
sessionId: "6ce27a04-7422-4687-9de9-138e68f308d8",
text: "Phase 2.5 (DX review) is done: DX score rose from 5.1 to 8.0/10, all 19 findings reviewed with 12 items accepted into the plan and one taste item flagged for gating. All pre-checks pass, so I'm moving on to Phase 3, the final Engineering Review of the amended plan.\n\n",
timestamp: "2026-09-10T00:16:29.888Z",
};
const at = Date.parse(announcement3.timestamp);
expect(autoplanPhaseCompletions({ status: 'ready', calls: [],
assistantMessages: [announcement3] }, at - 1)).toEqual([{ phase: 2.5, ts: at }]);
});
test('completed-state declarations retain supported phases, punctuation and first native time', () => {
for (const state of ['wrapped up', 'done']) for (const phase of [1, 2, 2.5, 3]) for (const tail of ['', '.', ': Work retained.', '. Work retained.']) {
expect(autoplanPhaseCompletions(transcript([1, `Phase ${phase} ${state}${tail}`]), START))
.toEqual([{ phase, ts: START + 1 }]);
}
expect(autoplanPhaseCompletions(transcript([1, '**Phase 1 wrapped up.**']), START))
.toEqual([{ phase: 1, ts: START + 1 }]);
expect(autoplanPhaseCompletions(transcript([4, 'Phase 1 wrapped up.'],
[2, 'Phase 3 wrapped up.'], [3, 'Phase 1 wrapped up: Moving to Phase 2.']), START))
.toEqual([{ phase: 3, ts: START + 2 }, { phase: 1, ts: START + 3 }]);
});
test('affirmative completion words share punctuation and optional is without future tense', () => {
for (const state of ['complete', 'completed', 'done', 'finished', 'wrapped up']) {
for (const copula of ['', 'is ']) for (const tail of ['', '.', ': Work retained.']) {
expect(autoplanPhaseCompletions(transcript([1, `Phase 3 ${copula}${state}${tail}`]), START))
.toEqual([{ phase: 3, ts: START + 1 }]);
}
for (const text of [`Phase 3 ${state}?`, `Phase 3 will be ${state}.`,
`Phase 3 is not ${state}.`, `Phase 3 ${state} if the reviewer finishes.`,
`Phase 3 ${state} when the work ends.`, `**Phase 3 ${state}** if approved.`,
`Example:\nPhase 3 ${state}.`, `> Phase 3 ${state}.`,
`Phase 3 (Eng review) is not ${state}.`, `Phase 3 (Eng review) ${state} if approved.`]) {
expect(autoplanPhaseCompletions(transcript([1, text]), START), text).toEqual([]);
}
}
});
test('known phase names must agree with their number and never supply completion alone', () => {
const names = [[1, 'CEO'], [2, 'Design'], [2.5, 'DX'], [3, 'Eng'], [3, 'Engineering']] as const;
for (const [phase, name] of names) for (const suffix of ['', ' review']) {
const declaration = `Phase ${phase} (${name}${suffix}) is finished.`;
expect(autoplanPhaseCompletions(transcript([1, declaration]), START)).toEqual([{ phase, ts: START + 1 }]);
expect(autoplanPhaseCompletions(transcript([1, `**${declaration}**`]), START)).toEqual([{ phase, ts: START + 1 }]);
for (const other of [1, 2, 2.5, 3].filter(n => n !== phase)) {
expect(autoplanPhaseCompletions(transcript([1, declaration.replace(`Phase ${phase}`, `Phase ${other}`)]), START))
.toEqual([]);
}
}
for (const text of ['Phase 3 (Eng review).', 'Phase 2.5 (future DX review) is done.',
'Phase 2.5 (DX review if approved) is done.', 'Phase 1 (source) complete.',
'Phase 2.5 ((DX review)) is done.', 'Phase 2.5 (DX review) finished soon.',
'# Phase 2.5 (DX review) is done.', 'Example:\nPhase 2.5 (DX review) is done.']) {
expect(autoplanPhaseCompletions(transcript([1, text]), START), text).toEqual([]);
}
});
test('future, conditional, negative and quoted wrap-up claims do not complete a phase', () => {
for (const text of [
'Phase 1 will wrap up.', 'Phase 1 has not wrapped up.', 'Phase 1 is not wrapped up.',
'Phase 1 wrapped up if the reviewer finishes.', 'Phase 1 wrapped up when the review ends.',
'Phase 1 wrapped up but is not complete.', 'Phase 1 wrapped up?',
'Once Phase 1 wrapped up, we would start Phase 2.', 'I will announce Phase 1 wrapped up.',
'**Phase 1 wrapped up** if the tests pass.', 'Phase 4 wrapped up.', 'Phase 2.1 wrapped up.',
'# Phase 1 wrapped up.', '> Phase 1 wrapped up.', '"Phase 1 wrapped up."',
'- Phase 1 wrapped up.', '| Phase 1 wrapped up. |', ' Phase 1 wrapped up.',
'```text\nPhase 1 wrapped up.\n```', '~~~text\nPhase 1 wrapped up.\n~~~',
'Example:\nPhase 1 wrapped up.\nPhase 2 wrapped up.',
'The template says:\n\nPhase 1 wrapped up.',
'**Phase 1 wrapped up.** Emit phase-transition summary:',
]) for (const declaration of [text, text.replace(/wrapped up/g, 'done')]) {
expect(autoplanPhaseCompletions(transcript([1, declaration]), START), declaration).toEqual([]);
}
});
test('wrapped-up declarations retain ready transcript and native timestamp requirements', () => {
const current = transcript([1, 'Phase 1 wrapped up.']);
for (const status of ['missing', 'error'] as const) {
expect(autoplanPhaseCompletions({ ...current, status }, START)).toEqual([]);
}
expect(autoplanPhaseCompletions(current, START + 2)).toEqual([]);
expect(autoplanPhaseCompletions({ ...current, assistantMessages: current.assistantMessages.map(
message => ({ ...message, timestamp: 'invalid' })) }, START)).toEqual([]);
});
test('retains actual completion timestamps when several phases arrive between polls', () => {
expect(autoplanPhaseCompletions(transcript(
[1, '**Phase 1 complete.** Codex: 2 concerns. Native: 3 issues.'],
[2, 'Phase 2 complete. Design outputs are in the plan.'],
[3, '**Phase 2.5 complete.** DX overall: 8/10.'],
[4, 'Phase 3 complete. Both engineering reviews finished.'],
), START)).toEqual([1, 2, 2.5, 3].map((phase, index) => ({ phase, ts: START + index + 1 })));
});
test('duplicate announcements retain their first timestamp without reordering phases', () => {
expect(autoplanPhaseCompletions(transcript(
[3, 'Phase 1 complete.'], [2, 'Phase 2 complete.'],
[1, '**Phase 1 complete.**'], [4, '**Phase 3 complete.**'],
), START)).toEqual([{ phase: 1, ts: START + 1 }, { phase: 2, ts: START + 2 }, { phase: 3, ts: START + 4 }]);
});
test('headings, quoted skill text, examples, tables and planned checklists provide no completion', () => {
for (const text of [
'## Phase 3 complete.',
'**PHASE 3 COMPLETE.** Emit phase-transition summary:',
'> **Phase 3 complete.** Codex: [N concerns].',
'The section says "**Phase 3 complete.**".',
'Example: **Phase 3 complete.**',
'Example announcement:\n**Phase 3 complete.**',
'Example announcement:\nPhase 2 complete.\nPhase 3 complete.',
'The template requires this completion marker:\n\n**Phase 3 complete.**',
' **Phase 3 complete.**',
'\tPhase 3 complete.',
'```markdown\n**Phase 3 complete.**\n```',
'```markdown\n```still-code\nPhase 3 complete.\n```',
'````markdown\n```\nPhase 3 complete.\n````',
'~~~markdown\nPhase 3 complete.\n~~~',
'```markdown\nPhase 3 complete.',
'| **Phase 3 complete.** | pending |',
'- [ ] **Phase 3 complete.**',
'1. Phase 3 complete.',
'I will announce **Phase 3 complete.** after the review.',
'Phase 3 complete when the engineering review ends.',
'Phase 3 complete?',
'Phase 4 complete.',
]) expect(autoplanPhaseCompletions(transcript([1, text]), START), text).toEqual([]);
});
test('a real announcement after a closed example fence still establishes completion', () => {
expect(autoplanPhaseCompletions(transcript([1, 'Example:\n```markdown\nPhase 3 complete.\n```\n\n**Phase 1 complete.**']), START))
.toEqual([{ phase: 1, ts: START + 1 }]);
});
test('accepts a template-compliant quoted transition with actual consensus while rejecting quoted examples', () => {
const actual = '> **Phase 1 complete.** Codex: 2 concerns. Claude subagent: 3 issues.\n' +
'> Consensus: 4/6 confirmed, 2 disagreements → surfaced at gate.\n> Passing to Phase 2.';
expect(autoplanPhaseCompletions(transcript([1, actual]), START)).toEqual([{ phase: 1, ts: START + 1 }]);
expect(autoplanPhaseCompletions(transcript([1, actual.replace('Codex: 2 concerns.', 'Codex: unavailable.')]), START))
.toEqual([{ phase: 1, ts: START + 1 }]);
for (const quoted of [
actual.replace('4/6', 'X/6'),
actual.replace('2 concerns', '[N concerns]'),
'The template contains this example:\n' + actual,
'Example announcement:\n' + actual + '\n' + actual.replace('Phase 1', 'Phase 3'),
'> **Phase 3 complete.**',
]) expect(autoplanPhaseCompletions(transcript([1, quoted]), START), quoted).toEqual([]);
});
test('missing/error transcripts and pre-command declarations cannot establish coverage', () => {
const prior = transcript([-1, '**Phase 3 complete.**']);
expect(autoplanPhaseCompletions(prior, START)).toEqual([]);
for (const status of ['missing', 'error'] as const) {
expect(autoplanPhaseCompletions({ ...transcript([1, '**Phase 3 complete.**']), status }, START)).toEqual([]);
}
});
test('does not repair an out-of-order chain or synthesize omitted phases', () => {
expect(autoplanPhaseCompletions(transcript([1, 'Phase 3 complete.'], [2, 'Phase 1 complete.']), START))
.toEqual([{ phase: 3, ts: START + 1 }, { phase: 1, ts: START + 2 }]);
expect(autoplanPhaseCompletions(transcript([1, 'Phase 2 skipped — no UI scope.']), START)).toEqual([]);
});
test('phase observer changes select the autoplan eval', () => {
for (const file of ['test/helpers/autoplan-phase-observer.ts', 'test/autoplan-phase-observer.test.ts']) {
expect(selectTests([file], E2E_TOUCHFILES).selected).toEqual(['autoplan-chain-pty']);
}
});
test.skipIf(process.platform === 'win32')('ANSI-rendered completions use native evidence while displayed Read/source markers do not', async () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'autoplan-phase-replay-'));
const fake = path.join(dir, 'fake-claude');
const worker = path.join(dir, 'worker.ts');
const recordFile = path.join(dir, 'events.jsonl');
const resultFile = path.join(dir, 'result.json');
fs.writeFileSync(fake, `#!${process.execPath}\n` + String.raw`
import * as fs from 'node:fs';
import * as path from 'node:path';
fs.writeFileSync(process.env.PHASE_RECORD, JSON.stringify({pid:process.pid}) + '\n');
const sessionId = 'fake-autoplan-session';
const dir = path.join(process.env.CLAUDE_CONFIG_DIR, 'projects', 'fixture');
fs.mkdirSync(dir, {recursive:true});
const base = Date.now();
const write = (role, content, offset, extra = {}) => fs.appendFileSync(path.join(dir, sessionId + '.jsonl'), JSON.stringify({
sessionId, cwd:process.cwd(), isSidechain:false, timestamp:new Date(base + offset).toISOString(), message:{role,content}, ...extra,
}) + '\n');
write('user', [{type:'tool_result', tool_use_id:'read', content:'**Phase 3 complete.**'}], 0);
write('assistant', [{type:'text', text:'> **Phase 3 complete.** is the quoted source marker.'}], 0);
write('assistant', [{type:'text', text:'**Phase 3 wrapped up.**'}], 0, {isSidechain:true});
write('assistant', [{type:'text', text:'**Phase 3 wrapped up.**'}], 0, {cwd:path.join(process.cwd(), 'foreign')});
process.stdin.setRawMode?.(true);
let sent = false;
process.stdin.on('data', data => {
if (sent || !data.toString().includes('\r')) return;
sent = true;
process.stdout.write('\x1b[2J\x1b[H');
[1, 2, 2.5, 3].forEach((phase, index) => {
const message = '**Phase ' + phase + (phase === 1 ? ' wrapped up.**' : phase === 2 ? ' done.**' : phase === 2.5 ? ' (DX review) is finished.**' : ' complete.**');
write('assistant', [{type:'text', text:message}], index + 1);
process.stdout.write('● \x1b[1mPhase ' + phase + ' complete.\x1b[22m\n');
});
process.stdout.write('NATIVE_PHASES_READY\n');
});
process.stdout.write('Read: autoplan/sections/eng-phase.md\n> **Phase 3 complete.**\nSOURCE_READY\n');
process.on('SIGINT', () => process.exit(0));
process.stdin.resume();
`);
fs.chmodSync(fake, 0o755);
const moduleUrl = (name: string) => pathToFileURL(path.resolve(import.meta.dir, 'helpers', name)).href;
fs.writeFileSync(worker, `
import { launchClaudePty } from ${JSON.stringify(moduleUrl('claude-pty-runner.ts'))};
import { readPlanCountTranscript } from ${JSON.stringify(moduleUrl('plan-count-transcript.ts'))};
import { autoplanPhaseCompletions } from ${JSON.stringify(moduleUrl('autoplan-phase-observer.ts'))};
const start = Date.now();
const session = await launchClaudePty({cwd:${JSON.stringify(dir)}, timeoutMs:8000, env:{PHASE_RECORD:${JSON.stringify(recordFile)}}});
try {
await session.waitFor('SOURCE_READY', {timeoutMs:4000, pollMs:20});
const read = () => readPlanCountTranscript(session.hermeticConfigDir, ${JSON.stringify(dir)});
const sourceOnly = autoplanPhaseCompletions(read(), start);
const oldPattern = /\\*\\*Phase\\s+(\\d+(?:\\.\\d+)?)\\s+complete\\.?\\*\\*/g;
const sourceFalsePositive = [...session.visibleText().matchAll(oldPattern)].length;
const since = session.mark();
session.send('\\r');
await session.waitFor('NATIVE_PHASES_READY', {timeoutMs:4000, pollMs:20});
const rendered = session.visibleSince(since);
const oldPatternMissed = [...rendered.matchAll(oldPattern)].length;
const hits = autoplanPhaseCompletions(read(), start);
await Bun.write(${JSON.stringify(resultFile)}, JSON.stringify({sourceOnly, sourceFalsePositive, oldPatternMissed, hits, rendered}));
} finally { await session.close(); }
`);
const child = Bun.spawn([process.execPath, worker], {
env: { ...process.env, BROWSE_TERMINAL_BINARY: fake, EVALS_HERMETIC: '1' }, stdout: 'pipe', stderr: 'pipe',
});
const timer = setTimeout(() => child.kill('SIGKILL'), 12_000);
try {
const [code, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]);
expect(code, stdout + stderr).toBe(0);
const result = JSON.parse(fs.readFileSync(resultFile, 'utf8'));
expect(result.sourceOnly).toEqual([]);
expect(result.sourceFalsePositive).toBe(1);
expect(result.oldPatternMissed).toBe(0);
expect(result.rendered).toContain('Phase 3 complete.');
expect(result.hits.map((hit: { phase: number }) => hit.phase)).toEqual([1, 2, 2.5, 3]);
for (let index = 1; index < result.hits.length; index++) {
expect(result.hits[index].ts).toBeGreaterThan(result.hits[index - 1].ts);
}
const pid = JSON.parse(fs.readFileSync(recordFile, 'utf8').split('\n')[0]!).pid;
expect(() => process.kill(pid, 0)).toThrow();
} finally {
clearTimeout(timer); child.kill('SIGKILL');
if (fs.existsSync(recordFile)) {
const pid = JSON.parse(fs.readFileSync(recordFile, 'utf8').split('\n')[0]!).pid;
try { process.kill(pid, 'SIGKILL'); } catch { /* already reaped */ }
}
fs.rmSync(dir, {recursive:true, force:true});
}
}, 15_000);
});