import { describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { copyPlanCountEvidence, createPlanCountSnapshotWriter, persistPlanCountSnapshot } from './helpers/plan-count-artifacts'; const input = { skillName: 'plan-design-review', observation: { outcome: 'timeout', reviewCount: 3 }, raw: '\x1b[2JSTART\r' + 'raw-frame\r'.repeat(2000) + '\x1b[31mEND', visible: 'START\r' + 'visible-frame\r'.repeat(2000) + 'END', cwd: '/temporary/count-fixture', claudeConfigDir: '/temporary/claude-config', }; describe('plan-count diagnostic artifacts', () => { test('refreshes one in-progress attempt through final outcome without keeping old copies', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-')); try { const save = createPlanCountSnapshotWriter({ EVALS_RUN_ID: 'checkpoint', GSTACK_EVAL_DIR: root }); const progress = save({ ...input, observation: { state: 'in_progress', reviewCount: 1 } }); const recordPath = path.join(progress.artifactDir!, 'observation.json'); expect(JSON.parse(fs.readFileSync(recordPath, 'utf8'))).toMatchObject({ state: 'in_progress', reviewCount: 1 }); const final = save({ ...input, raw: input.raw + '\nlast question', observation: { outcome: 'completion_summary', reviewCount: 5 } }); expect(final.artifactDir).toBe(progress.artifactDir); const record = JSON.parse(fs.readFileSync(recordPath, 'utf8')); expect(record).toMatchObject({ outcome: 'completion_summary', reviewCount: 5 }); expect(record.state).toBeUndefined(); expect(fs.readdirSync(final.artifactDir!).sort()).toEqual(['observation.json', 'terminal.raw.log', 'terminal.visible.log']); expect(fs.readFileSync(path.join(final.artifactDir!, 'terminal.raw.log'), 'utf8')).toBe(input.raw + '\nlast question'); } finally { fs.rmSync(root, { recursive: true, force: true }); } }); test('keeps complete output and distinct retry records in the configured eval directory', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-')); try { const env = { EVALS_RUN_ID: 'same-run', GSTACK_EVAL_DIR: root }; const first = persistPlanCountSnapshot(input, env); const second = persistPlanCountSnapshot({ ...input, raw: 'retry' }, env); expect(first.artifactError).toBeUndefined(); expect(second.artifactError).toBeUndefined(); expect(first.artifactDir).not.toBe(second.artifactDir); expect(path.relative(root, first.artifactDir!)).toStartWith(path.join('pty-count', 'same-run')); expect(fs.readFileSync(path.join(first.artifactDir!, 'terminal.raw.log'), 'utf8')).toBe(input.raw); expect(fs.readFileSync(path.join(first.artifactDir!, 'terminal.visible.log'), 'utf8')).toBe(input.visible); const record = JSON.parse(fs.readFileSync(path.join(first.artifactDir!, 'observation.json'), 'utf8')); expect(record).toMatchObject({ ...input.observation, artifactDir: first.artifactDir, capture: { skill: input.skillName, cwd: input.cwd, claudeConfigDir: input.claudeConfigDir } }); expect(fs.readFileSync(path.join(second.artifactDir!, 'terminal.raw.log'), 'utf8')).toBe('retry'); if (process.platform !== 'win32') { expect(fs.statSync(path.join(first.artifactDir!, 'terminal.raw.log')).mode & 0o777).toBe(0o600); } } finally { fs.rmSync(root, { recursive: true, force: true }); } }); test('retains explicitly requested captures without a run ID and isolates invocation fallbacks', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-')); try { // The paid shard launcher sets GSTACK_EVAL_DIR without EVALS_RUN_ID. const env = { GSTACK_EVAL_DIR: root }; const save = createPlanCountSnapshotWriter(env); const progress = save({ ...input, observation: { state: 'in_progress', reviewCount: 1 } }); expect(progress.artifactError).toBeUndefined(); expect(progress.artifactDir).toBeDefined(); const first = JSON.parse(fs.readFileSync(path.join(progress.artifactDir!, 'observation.json'), 'utf8')); expect(first.capture.runId).toMatch(/^local-[0-9a-f-]{36}$/); expect(path.relative(root, progress.artifactDir!)).toStartWith(path.join('pty-count', first.capture.runId)); const final = save({ ...input, raw: 'final output', observation: { outcome: 'plan_ready', reviewCount: 5 } }); expect(final.artifactDir).toBe(progress.artifactDir); const record = JSON.parse(fs.readFileSync(path.join(final.artifactDir!, 'observation.json'), 'utf8')); expect(record).toMatchObject({ outcome: 'plan_ready', capture: { runId: first.capture.runId } }); expect(record.state).toBeUndefined(); expect(fs.readFileSync(path.join(final.artifactDir!, 'terminal.raw.log'), 'utf8')).toBe('final output'); expect(fs.readFileSync(path.join(final.artifactDir!, 'terminal.visible.log'), 'utf8')).toBe(input.visible); const retry = persistPlanCountSnapshot(input, env); const retried = JSON.parse(fs.readFileSync(path.join(retry.artifactDir!, 'observation.json'), 'utf8')); expect(retry.artifactDir).not.toBe(final.artifactDir); expect(retried.capture.runId).not.toBe(first.capture.runId); expect(path.dirname(retry.artifactDir!)).not.toBe(path.dirname(final.artifactDir!)); } finally { fs.rmSync(root, { recursive: true, force: true }); } }); test('does not persist ordinary free runs without either capture setting', () => { expect(persistPlanCountSnapshot(input, {})).toEqual({}); expect(persistPlanCountSnapshot(input, { EVALS_RUN_ID: '', GSTACK_EVAL_DIR: '' })).toEqual({}); }); test('keeps run and skill identifiers within the owned artifact directory', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-')); try { const saved = persistPlanCountSnapshot({ ...input, skillName: '../../other' }, { EVALS_RUN_ID: '../outside', GSTACK_EVAL_DIR: root, }); expect(saved.artifactError).toBeUndefined(); expect(path.relative(root, saved.artifactDir!)).not.toStartWith('..'); } finally { fs.rmSync(root, { recursive: true, force: true }); } }); test('reports a diagnostic write failure without replacing the original observation', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-')); const file = path.join(root, 'file'); fs.writeFileSync(file, 'unchanged'); try { const result = persistPlanCountSnapshot(input, { EVALS_RUN_ID: 'run', GSTACK_EVAL_DIR: file }); expect(result.artifactError).toBeDefined(); expect(input.observation).toEqual({ outcome: 'timeout', reviewCount: 3 }); expect(fs.readFileSync(file, 'utf8')).toBe('unchanged'); } finally { fs.rmSync(root, { recursive: true, force: true }); } }); test('copies the plan and review-log rows so they survive fixture cleanup', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-')); const fixture = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-fixture-')); try { const planPath = path.join(fixture, 'plan.md'); const logs = path.join(fixture, 'state', 'projects', 'slug'); fs.mkdirSync(logs, { recursive: true }); fs.writeFileSync(planPath, '# plan\n## GSTACK REVIEW REPORT\n'); fs.writeFileSync(path.join(logs, 'main-reviews.jsonl'), '{"skill":"plan-design-review","status":"clean"}\n'); fs.writeFileSync(path.join(logs, 'unrelated.json'), '{}'); const result = persistPlanCountSnapshot(input, { EVALS_RUN_ID: 'run', GSTACK_EVAL_DIR: root }); copyPlanCountEvidence(result.artifactDir, { planPath, reviewLogDirectory: logs }); fs.rmSync(fixture, { recursive: true, force: true }); expect(result.artifactError).toBeUndefined(); expect(fs.readFileSync(path.join(result.artifactDir!, 'evidence', 'plan', 'plan.md'), 'utf8')).toBe('# plan\n## GSTACK REVIEW REPORT\n'); expect(fs.readFileSync(path.join(result.artifactDir!, 'evidence', 'review-log', 'main-reviews.jsonl'), 'utf8')) .toBe('{"skill":"plan-design-review","status":"clean"}\n'); expect(fs.existsSync(path.join(result.artifactDir!, 'evidence', 'review-log', 'unrelated.json'))).toBe(false); expect(JSON.parse(fs.readFileSync(path.join(result.artifactDir!, 'evidence-copy.json'), 'utf8'))).toEqual({ plan: `copied: ${planPath}`, 'reviewLog:main-reviews.jsonl': `copied: ${path.join(logs, 'main-reviews.jsonl')}`, }); expect(JSON.parse(fs.readFileSync(path.join(result.artifactDir!, 'observation.json'), 'utf8')).outcome).toBe('timeout'); } finally { fs.rmSync(root, { recursive: true, force: true }); fs.rmSync(fixture, { recursive: true, force: true }); } }); test('a missing plan or unreadable log is recorded and never replaces the observation', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-')); try { const planPath = path.join(root, 'never-written.md'); const notDirectory = path.join(root, 'not-a-directory'); fs.writeFileSync(notDirectory, 'file'); const result = persistPlanCountSnapshot(input, { EVALS_RUN_ID: 'run', GSTACK_EVAL_DIR: path.join(root, 'evals') }); expect(() => copyPlanCountEvidence(result.artifactDir, { planPath, reviewLogDirectory: notDirectory })).not.toThrow(); expect(result.artifactError).toBeUndefined(); const copy = JSON.parse(fs.readFileSync(path.join(result.artifactDir!, 'evidence-copy.json'), 'utf8')); expect(copy.plan).toBe(`missing: ${planPath}`); expect(copy.reviewLog).toStartWith('error: '); expect(JSON.parse(fs.readFileSync(path.join(result.artifactDir!, 'observation.json'), 'utf8'))) .toMatchObject({ outcome: 'timeout', reviewCount: 3 }); const bare = persistPlanCountSnapshot(input, { EVALS_RUN_ID: 'run', GSTACK_EVAL_DIR: path.join(root, 'evals') }); copyPlanCountEvidence(bare.artifactDir, {}); expect(() => copyPlanCountEvidence(undefined, { planPath })).not.toThrow(); expect(JSON.parse(fs.readFileSync(path.join(bare.artifactDir!, 'evidence-copy.json'), 'utf8'))).toEqual({ plan: 'missing: no expected plan path', reviewLog: 'missing: no fixture-owned review log binding' }); } finally { fs.rmSync(root, { recursive: true, force: true }); } }); });