mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-16 13:21:30 +01:00
301 lines
17 KiB
TypeScript
301 lines
17 KiB
TypeScript
import { describe, expect, test } from 'bun:test';
|
|
import * as fs from 'node:fs';
|
|
import * as os from 'node:os';
|
|
import * as path from 'node:path';
|
|
import { pathToFileURL } from 'node:url';
|
|
import { autoplanPhaseCompletions } from './helpers/autoplan-phase-observer';
|
|
import type { PlanCountTranscript } from './helpers/plan-count-transcript';
|
|
import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles';
|
|
|
|
const START = Date.parse('2026-09-08T16:00:00.000Z');
|
|
const transcript = (...messages: Array<[number, string]>): PlanCountTranscript => ({
|
|
status: 'ready', calls: [],
|
|
assistantMessages: messages.map(([ms, text]) => ({
|
|
sessionId: 'autoplan-fixture', timestamp: new Date(START + ms).toISOString(), text,
|
|
})),
|
|
});
|
|
|
|
describe('native autoplan phase observation', () => {
|
|
test('AF public wrapped-up announcement retains its actual Phase 1 timestamp', () => {
|
|
// Exact reader-projected public parent narration from the AF run; no raw
|
|
// signature or private block is stored. The next-phase mention adds no hit.
|
|
const announcement = {
|
|
sessionId: '6ce27a04-7422-4687-9de9-138e68f308d8',
|
|
text: 'Phase 1 wrapped up: 11 findings from the Claude subagent, 30 of 34 spec issues fixed after 3 review rounds, and 21 obligations carried forward with 3 disagreements flagged as taste items. Moving on to Phase 2 (design review) now that UI scope was detected.\n\n',
|
|
timestamp: '2026-09-10T00:00:03.893Z',
|
|
};
|
|
const at = Date.parse(announcement.timestamp);
|
|
expect(autoplanPhaseCompletions({ status: 'ready', calls: [],
|
|
assistantMessages: [announcement] }, at - 1)).toEqual([{ phase: 1, ts: at }]);
|
|
});
|
|
|
|
test('AF bare done announcement retains Phase 2 without crediting its DX transition', () => {
|
|
const announcement2 = {
|
|
sessionId: "6ce27a04-7422-4687-9de9-138e68f308d8",
|
|
text: "Phase 2 done: Claude subagent found 15 issues (1 critical, 8 high, 6 medium), with 14 fully accepted and 1 partially accepted; design score rose from 6/10 to 8.7/10, and 10 accepted items are now carried into the Implementation plan. Moving on to Phase 2.5 (DX Review) since developer-facing scope was detected.",
|
|
timestamp: "2026-09-10T00:08:31.892Z",
|
|
};
|
|
const at = Date.parse(announcement2.timestamp);
|
|
expect(autoplanPhaseCompletions({ status: 'ready', calls: [],
|
|
assistantMessages: [announcement2] }, at - 1)).toEqual([{ phase: 2, ts: at }]);
|
|
});
|
|
|
|
test('AF named DX completion retains Phase 2.5 without crediting its Eng transition', () => {
|
|
const announcement3 = {
|
|
sessionId: "6ce27a04-7422-4687-9de9-138e68f308d8",
|
|
text: "Phase 2.5 (DX review) is done: DX score rose from 5.1 to 8.0/10, all 19 findings reviewed with 12 items accepted into the plan and one taste item flagged for gating. All pre-checks pass, so I'm moving on to Phase 3, the final Engineering Review of the amended plan.\n\n",
|
|
timestamp: "2026-09-10T00:16:29.888Z",
|
|
};
|
|
const at = Date.parse(announcement3.timestamp);
|
|
expect(autoplanPhaseCompletions({ status: 'ready', calls: [],
|
|
assistantMessages: [announcement3] }, at - 1)).toEqual([{ phase: 2.5, ts: at }]);
|
|
});
|
|
|
|
test('completed-state declarations retain supported phases, punctuation and first native time', () => {
|
|
for (const state of ['wrapped up', 'done']) for (const phase of [1, 2, 2.5, 3]) for (const tail of ['', '.', ': Work retained.', '. Work retained.']) {
|
|
expect(autoplanPhaseCompletions(transcript([1, `Phase ${phase} ${state}${tail}`]), START))
|
|
.toEqual([{ phase, ts: START + 1 }]);
|
|
}
|
|
expect(autoplanPhaseCompletions(transcript([1, '**Phase 1 wrapped up.**']), START))
|
|
.toEqual([{ phase: 1, ts: START + 1 }]);
|
|
expect(autoplanPhaseCompletions(transcript([4, 'Phase 1 wrapped up.'],
|
|
[2, 'Phase 3 wrapped up.'], [3, 'Phase 1 wrapped up: Moving to Phase 2.']), START))
|
|
.toEqual([{ phase: 3, ts: START + 2 }, { phase: 1, ts: START + 3 }]);
|
|
});
|
|
|
|
test('affirmative completion words share punctuation and optional is without future tense', () => {
|
|
for (const state of ['complete', 'completed', 'done', 'finished', 'wrapped up']) {
|
|
for (const copula of ['', 'is ']) for (const tail of ['', '.', ': Work retained.']) {
|
|
expect(autoplanPhaseCompletions(transcript([1, `Phase 3 ${copula}${state}${tail}`]), START))
|
|
.toEqual([{ phase: 3, ts: START + 1 }]);
|
|
}
|
|
for (const text of [`Phase 3 ${state}?`, `Phase 3 will be ${state}.`,
|
|
`Phase 3 is not ${state}.`, `Phase 3 ${state} if the reviewer finishes.`,
|
|
`Phase 3 ${state} when the work ends.`, `**Phase 3 ${state}** if approved.`,
|
|
`Example:\nPhase 3 ${state}.`, `> Phase 3 ${state}.`,
|
|
`Phase 3 (Eng review) is not ${state}.`, `Phase 3 (Eng review) ${state} if approved.`]) {
|
|
expect(autoplanPhaseCompletions(transcript([1, text]), START), text).toEqual([]);
|
|
}
|
|
}
|
|
});
|
|
|
|
test('known phase names must agree with their number and never supply completion alone', () => {
|
|
const names = [[1, 'CEO'], [2, 'Design'], [2.5, 'DX'], [3, 'Eng'], [3, 'Engineering']] as const;
|
|
for (const [phase, name] of names) for (const suffix of ['', ' review']) {
|
|
const declaration = `Phase ${phase} (${name}${suffix}) is finished.`;
|
|
expect(autoplanPhaseCompletions(transcript([1, declaration]), START)).toEqual([{ phase, ts: START + 1 }]);
|
|
expect(autoplanPhaseCompletions(transcript([1, `**${declaration}**`]), START)).toEqual([{ phase, ts: START + 1 }]);
|
|
for (const other of [1, 2, 2.5, 3].filter(n => n !== phase)) {
|
|
expect(autoplanPhaseCompletions(transcript([1, declaration.replace(`Phase ${phase}`, `Phase ${other}`)]), START))
|
|
.toEqual([]);
|
|
}
|
|
}
|
|
for (const text of ['Phase 3 (Eng review).', 'Phase 2.5 (future DX review) is done.',
|
|
'Phase 2.5 (DX review if approved) is done.', 'Phase 1 (source) complete.',
|
|
'Phase 2.5 ((DX review)) is done.', 'Phase 2.5 (DX review) finished soon.',
|
|
'# Phase 2.5 (DX review) is done.', 'Example:\nPhase 2.5 (DX review) is done.']) {
|
|
expect(autoplanPhaseCompletions(transcript([1, text]), START), text).toEqual([]);
|
|
}
|
|
});
|
|
|
|
test('future, conditional, negative and quoted wrap-up claims do not complete a phase', () => {
|
|
for (const text of [
|
|
'Phase 1 will wrap up.', 'Phase 1 has not wrapped up.', 'Phase 1 is not wrapped up.',
|
|
'Phase 1 wrapped up if the reviewer finishes.', 'Phase 1 wrapped up when the review ends.',
|
|
'Phase 1 wrapped up but is not complete.', 'Phase 1 wrapped up?',
|
|
'Once Phase 1 wrapped up, we would start Phase 2.', 'I will announce Phase 1 wrapped up.',
|
|
'**Phase 1 wrapped up** if the tests pass.', 'Phase 4 wrapped up.', 'Phase 2.1 wrapped up.',
|
|
'# Phase 1 wrapped up.', '> Phase 1 wrapped up.', '"Phase 1 wrapped up."',
|
|
'- Phase 1 wrapped up.', '| Phase 1 wrapped up. |', ' Phase 1 wrapped up.',
|
|
'```text\nPhase 1 wrapped up.\n```', '~~~text\nPhase 1 wrapped up.\n~~~',
|
|
'Example:\nPhase 1 wrapped up.\nPhase 2 wrapped up.',
|
|
'The template says:\n\nPhase 1 wrapped up.',
|
|
'**Phase 1 wrapped up.** Emit phase-transition summary:',
|
|
]) for (const declaration of [text, text.replace(/wrapped up/g, 'done')]) {
|
|
expect(autoplanPhaseCompletions(transcript([1, declaration]), START), declaration).toEqual([]);
|
|
}
|
|
});
|
|
|
|
test('wrapped-up declarations retain ready transcript and native timestamp requirements', () => {
|
|
const current = transcript([1, 'Phase 1 wrapped up.']);
|
|
for (const status of ['missing', 'error'] as const) {
|
|
expect(autoplanPhaseCompletions({ ...current, status }, START)).toEqual([]);
|
|
}
|
|
expect(autoplanPhaseCompletions(current, START + 2)).toEqual([]);
|
|
expect(autoplanPhaseCompletions({ ...current, assistantMessages: current.assistantMessages.map(
|
|
message => ({ ...message, timestamp: 'invalid' })) }, START)).toEqual([]);
|
|
});
|
|
|
|
test('retains actual completion timestamps when several phases arrive between polls', () => {
|
|
expect(autoplanPhaseCompletions(transcript(
|
|
[1, '**Phase 1 complete.** Codex: 2 concerns. Native: 3 issues.'],
|
|
[2, 'Phase 2 complete. Design outputs are in the plan.'],
|
|
[3, '**Phase 2.5 complete.** DX overall: 8/10.'],
|
|
[4, 'Phase 3 complete. Both engineering reviews finished.'],
|
|
), START)).toEqual([1, 2, 2.5, 3].map((phase, index) => ({ phase, ts: START + index + 1 })));
|
|
});
|
|
|
|
test('duplicate announcements retain their first timestamp without reordering phases', () => {
|
|
expect(autoplanPhaseCompletions(transcript(
|
|
[3, 'Phase 1 complete.'], [2, 'Phase 2 complete.'],
|
|
[1, '**Phase 1 complete.**'], [4, '**Phase 3 complete.**'],
|
|
), START)).toEqual([{ phase: 1, ts: START + 1 }, { phase: 2, ts: START + 2 }, { phase: 3, ts: START + 4 }]);
|
|
});
|
|
|
|
test('headings, quoted skill text, examples, tables and planned checklists provide no completion', () => {
|
|
for (const text of [
|
|
'## Phase 3 complete.',
|
|
'**PHASE 3 COMPLETE.** Emit phase-transition summary:',
|
|
'> **Phase 3 complete.** Codex: [N concerns].',
|
|
'The section says "**Phase 3 complete.**".',
|
|
'Example: **Phase 3 complete.**',
|
|
'Example announcement:\n**Phase 3 complete.**',
|
|
'Example announcement:\nPhase 2 complete.\nPhase 3 complete.',
|
|
'The template requires this completion marker:\n\n**Phase 3 complete.**',
|
|
' **Phase 3 complete.**',
|
|
'\tPhase 3 complete.',
|
|
'```markdown\n**Phase 3 complete.**\n```',
|
|
'```markdown\n```still-code\nPhase 3 complete.\n```',
|
|
'````markdown\n```\nPhase 3 complete.\n````',
|
|
'~~~markdown\nPhase 3 complete.\n~~~',
|
|
'```markdown\nPhase 3 complete.',
|
|
'| **Phase 3 complete.** | pending |',
|
|
'- [ ] **Phase 3 complete.**',
|
|
'1. Phase 3 complete.',
|
|
'I will announce **Phase 3 complete.** after the review.',
|
|
'Phase 3 complete when the engineering review ends.',
|
|
'Phase 3 complete?',
|
|
'Phase 4 complete.',
|
|
]) expect(autoplanPhaseCompletions(transcript([1, text]), START), text).toEqual([]);
|
|
});
|
|
|
|
test('a real announcement after a closed example fence still establishes completion', () => {
|
|
expect(autoplanPhaseCompletions(transcript([1, 'Example:\n```markdown\nPhase 3 complete.\n```\n\n**Phase 1 complete.**']), START))
|
|
.toEqual([{ phase: 1, ts: START + 1 }]);
|
|
});
|
|
|
|
test('accepts a template-compliant quoted transition with actual consensus while rejecting quoted examples', () => {
|
|
const actual = '> **Phase 1 complete.** Codex: 2 concerns. Claude subagent: 3 issues.\n' +
|
|
'> Consensus: 4/6 confirmed, 2 disagreements → surfaced at gate.\n> Passing to Phase 2.';
|
|
expect(autoplanPhaseCompletions(transcript([1, actual]), START)).toEqual([{ phase: 1, ts: START + 1 }]);
|
|
expect(autoplanPhaseCompletions(transcript([1, actual.replace('Codex: 2 concerns.', 'Codex: unavailable.')]), START))
|
|
.toEqual([{ phase: 1, ts: START + 1 }]);
|
|
for (const quoted of [
|
|
actual.replace('4/6', 'X/6'),
|
|
actual.replace('2 concerns', '[N concerns]'),
|
|
'The template contains this example:\n' + actual,
|
|
'Example announcement:\n' + actual + '\n' + actual.replace('Phase 1', 'Phase 3'),
|
|
'> **Phase 3 complete.**',
|
|
]) expect(autoplanPhaseCompletions(transcript([1, quoted]), START), quoted).toEqual([]);
|
|
});
|
|
|
|
test('missing/error transcripts and pre-command declarations cannot establish coverage', () => {
|
|
const prior = transcript([-1, '**Phase 3 complete.**']);
|
|
expect(autoplanPhaseCompletions(prior, START)).toEqual([]);
|
|
for (const status of ['missing', 'error'] as const) {
|
|
expect(autoplanPhaseCompletions({ ...transcript([1, '**Phase 3 complete.**']), status }, START)).toEqual([]);
|
|
}
|
|
});
|
|
|
|
test('does not repair an out-of-order chain or synthesize omitted phases', () => {
|
|
expect(autoplanPhaseCompletions(transcript([1, 'Phase 3 complete.'], [2, 'Phase 1 complete.']), START))
|
|
.toEqual([{ phase: 3, ts: START + 1 }, { phase: 1, ts: START + 2 }]);
|
|
expect(autoplanPhaseCompletions(transcript([1, 'Phase 2 skipped — no UI scope.']), START)).toEqual([]);
|
|
});
|
|
|
|
test('phase observer changes select the autoplan eval', () => {
|
|
for (const file of ['test/helpers/autoplan-phase-observer.ts', 'test/autoplan-phase-observer.test.ts']) {
|
|
expect(selectTests([file], E2E_TOUCHFILES).selected).toEqual(['autoplan-chain-pty']);
|
|
}
|
|
});
|
|
|
|
test.skipIf(process.platform === 'win32')('ANSI-rendered completions use native evidence while displayed Read/source markers do not', async () => {
|
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'autoplan-phase-replay-'));
|
|
const fake = path.join(dir, 'fake-claude');
|
|
const worker = path.join(dir, 'worker.ts');
|
|
const recordFile = path.join(dir, 'events.jsonl');
|
|
const resultFile = path.join(dir, 'result.json');
|
|
fs.writeFileSync(fake, `#!${process.execPath}\n` + String.raw`
|
|
import * as fs from 'node:fs';
|
|
import * as path from 'node:path';
|
|
fs.writeFileSync(process.env.PHASE_RECORD, JSON.stringify({pid:process.pid}) + '\n');
|
|
const sessionId = 'fake-autoplan-session';
|
|
const dir = path.join(process.env.CLAUDE_CONFIG_DIR, 'projects', 'fixture');
|
|
fs.mkdirSync(dir, {recursive:true});
|
|
const base = Date.now();
|
|
const write = (role, content, offset, extra = {}) => fs.appendFileSync(path.join(dir, sessionId + '.jsonl'), JSON.stringify({
|
|
sessionId, cwd:process.cwd(), isSidechain:false, timestamp:new Date(base + offset).toISOString(), message:{role,content}, ...extra,
|
|
}) + '\n');
|
|
write('user', [{type:'tool_result', tool_use_id:'read', content:'**Phase 3 complete.**'}], 0);
|
|
write('assistant', [{type:'text', text:'> **Phase 3 complete.** is the quoted source marker.'}], 0);
|
|
write('assistant', [{type:'text', text:'**Phase 3 wrapped up.**'}], 0, {isSidechain:true});
|
|
write('assistant', [{type:'text', text:'**Phase 3 wrapped up.**'}], 0, {cwd:path.join(process.cwd(), 'foreign')});
|
|
process.stdin.setRawMode?.(true);
|
|
let sent = false;
|
|
process.stdin.on('data', data => {
|
|
if (sent || !data.toString().includes('\r')) return;
|
|
sent = true;
|
|
process.stdout.write('\x1b[2J\x1b[H');
|
|
[1, 2, 2.5, 3].forEach((phase, index) => {
|
|
const message = '**Phase ' + phase + (phase === 1 ? ' wrapped up.**' : phase === 2 ? ' done.**' : phase === 2.5 ? ' (DX review) is finished.**' : ' complete.**');
|
|
write('assistant', [{type:'text', text:message}], index + 1);
|
|
process.stdout.write('● \x1b[1mPhase ' + phase + ' complete.\x1b[22m\n');
|
|
});
|
|
process.stdout.write('NATIVE_PHASES_READY\n');
|
|
});
|
|
process.stdout.write('Read: autoplan/sections/eng-phase.md\n> **Phase 3 complete.**\nSOURCE_READY\n');
|
|
process.on('SIGINT', () => process.exit(0));
|
|
process.stdin.resume();
|
|
`);
|
|
fs.chmodSync(fake, 0o755);
|
|
const moduleUrl = (name: string) => pathToFileURL(path.resolve(import.meta.dir, 'helpers', name)).href;
|
|
fs.writeFileSync(worker, `
|
|
import { launchClaudePty } from ${JSON.stringify(moduleUrl('claude-pty-runner.ts'))};
|
|
import { readPlanCountTranscript } from ${JSON.stringify(moduleUrl('plan-count-transcript.ts'))};
|
|
import { autoplanPhaseCompletions } from ${JSON.stringify(moduleUrl('autoplan-phase-observer.ts'))};
|
|
const start = Date.now();
|
|
const session = await launchClaudePty({cwd:${JSON.stringify(dir)}, timeoutMs:8000, env:{PHASE_RECORD:${JSON.stringify(recordFile)}}});
|
|
try {
|
|
await session.waitFor('SOURCE_READY', {timeoutMs:4000, pollMs:20});
|
|
const read = () => readPlanCountTranscript(session.hermeticConfigDir, ${JSON.stringify(dir)});
|
|
const sourceOnly = autoplanPhaseCompletions(read(), start);
|
|
const oldPattern = /\\*\\*Phase\\s+(\\d+(?:\\.\\d+)?)\\s+complete\\.?\\*\\*/g;
|
|
const sourceFalsePositive = [...session.visibleText().matchAll(oldPattern)].length;
|
|
const since = session.mark();
|
|
session.send('\\r');
|
|
await session.waitFor('NATIVE_PHASES_READY', {timeoutMs:4000, pollMs:20});
|
|
const rendered = session.visibleSince(since);
|
|
const oldPatternMissed = [...rendered.matchAll(oldPattern)].length;
|
|
const hits = autoplanPhaseCompletions(read(), start);
|
|
await Bun.write(${JSON.stringify(resultFile)}, JSON.stringify({sourceOnly, sourceFalsePositive, oldPatternMissed, hits, rendered}));
|
|
} finally { await session.close(); }
|
|
`);
|
|
const child = Bun.spawn([process.execPath, worker], {
|
|
env: { ...process.env, BROWSE_TERMINAL_BINARY: fake, EVALS_HERMETIC: '1' }, stdout: 'pipe', stderr: 'pipe',
|
|
});
|
|
const timer = setTimeout(() => child.kill('SIGKILL'), 12_000);
|
|
try {
|
|
const [code, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]);
|
|
expect(code, stdout + stderr).toBe(0);
|
|
const result = JSON.parse(fs.readFileSync(resultFile, 'utf8'));
|
|
expect(result.sourceOnly).toEqual([]);
|
|
expect(result.sourceFalsePositive).toBe(1);
|
|
expect(result.oldPatternMissed).toBe(0);
|
|
expect(result.rendered).toContain('Phase 3 complete.');
|
|
expect(result.hits.map((hit: { phase: number }) => hit.phase)).toEqual([1, 2, 2.5, 3]);
|
|
for (let index = 1; index < result.hits.length; index++) {
|
|
expect(result.hits[index].ts).toBeGreaterThan(result.hits[index - 1].ts);
|
|
}
|
|
const pid = JSON.parse(fs.readFileSync(recordFile, 'utf8').split('\n')[0]!).pid;
|
|
expect(() => process.kill(pid, 0)).toThrow();
|
|
} finally {
|
|
clearTimeout(timer); child.kill('SIGKILL');
|
|
if (fs.existsSync(recordFile)) {
|
|
const pid = JSON.parse(fs.readFileSync(recordFile, 'utf8').split('\n')[0]!).pid;
|
|
try { process.kill(pid, 'SIGKILL'); } catch { /* already reaped */ }
|
|
}
|
|
fs.rmSync(dir, {recursive:true, force:true});
|
|
}
|
|
}, 15_000);
|
|
});
|