import { describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { createPlanCountSnapshotWriter, persistPlanCountSnapshot } from './helpers/plan-count-artifacts'; const input = { skillName: 'plan-design-review', observation: { outcome: 'timeout', reviewCount: 3 }, raw: '\x1b[2JSTART\r' + 'raw-frame\r'.repeat(2000) + '\x1b[31mEND', visible: 'START\r' + 'visible-frame\r'.repeat(2000) + 'END', cwd: '/temporary/count-fixture', claudeConfigDir: '/temporary/claude-config', }; describe('plan-count diagnostic artifacts', () => { test('refreshes one in-progress attempt through final outcome without keeping old copies', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-')); try { const save = createPlanCountSnapshotWriter({ EVALS_RUN_ID: 'checkpoint', GSTACK_EVAL_DIR: root }); const progress = save({ ...input, observation: { state: 'in_progress', reviewCount: 1 } }); const recordPath = path.join(progress.artifactDir!, 'observation.json'); expect(JSON.parse(fs.readFileSync(recordPath, 'utf8'))).toMatchObject({ state: 'in_progress', reviewCount: 1 }); const final = save({ ...input, raw: input.raw + '\nlast question', observation: { outcome: 'completion_summary', reviewCount: 5 } }); expect(final.artifactDir).toBe(progress.artifactDir); const record = JSON.parse(fs.readFileSync(recordPath, 'utf8')); expect(record).toMatchObject({ outcome: 'completion_summary', reviewCount: 5 }); expect(record.state).toBeUndefined(); expect(fs.readdirSync(final.artifactDir!).sort()).toEqual(['observation.json', 'terminal.raw.log', 'terminal.visible.log']); expect(fs.readFileSync(path.join(final.artifactDir!, 'terminal.raw.log'), 'utf8')).toBe(input.raw + '\nlast question'); } finally { fs.rmSync(root, { recursive: true, force: true }); } }); test('keeps complete output and distinct retry records in the configured eval directory', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-')); try { const env = { EVALS_RUN_ID: 'same-run', GSTACK_EVAL_DIR: root }; const first = persistPlanCountSnapshot(input, env); const second = persistPlanCountSnapshot({ ...input, raw: 'retry' }, env); expect(first.artifactError).toBeUndefined(); expect(second.artifactError).toBeUndefined(); expect(first.artifactDir).not.toBe(second.artifactDir); expect(path.relative(root, first.artifactDir!)).toStartWith(path.join('pty-count', 'same-run')); expect(fs.readFileSync(path.join(first.artifactDir!, 'terminal.raw.log'), 'utf8')).toBe(input.raw); expect(fs.readFileSync(path.join(first.artifactDir!, 'terminal.visible.log'), 'utf8')).toBe(input.visible); const record = JSON.parse(fs.readFileSync(path.join(first.artifactDir!, 'observation.json'), 'utf8')); expect(record).toMatchObject({ ...input.observation, artifactDir: first.artifactDir, capture: { skill: input.skillName, cwd: input.cwd, claudeConfigDir: input.claudeConfigDir } }); expect(fs.readFileSync(path.join(second.artifactDir!, 'terminal.raw.log'), 'utf8')).toBe('retry'); if (process.platform !== 'win32') { expect(fs.statSync(path.join(first.artifactDir!, 'terminal.raw.log')).mode & 0o777).toBe(0o600); } } finally { fs.rmSync(root, { recursive: true, force: true }); } }); test('retains explicitly requested captures without a run ID and isolates invocation fallbacks', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-')); try { // The paid shard launcher sets GSTACK_EVAL_DIR without EVALS_RUN_ID. const env = { GSTACK_EVAL_DIR: root }; const save = createPlanCountSnapshotWriter(env); const progress = save({ ...input, observation: { state: 'in_progress', reviewCount: 1 } }); expect(progress.artifactError).toBeUndefined(); expect(progress.artifactDir).toBeDefined(); const first = JSON.parse(fs.readFileSync(path.join(progress.artifactDir!, 'observation.json'), 'utf8')); expect(first.capture.runId).toMatch(/^local-[0-9a-f-]{36}$/); expect(path.relative(root, progress.artifactDir!)).toStartWith(path.join('pty-count', first.capture.runId)); const final = save({ ...input, raw: 'final output', observation: { outcome: 'plan_ready', reviewCount: 5 } }); expect(final.artifactDir).toBe(progress.artifactDir); const record = JSON.parse(fs.readFileSync(path.join(final.artifactDir!, 'observation.json'), 'utf8')); expect(record).toMatchObject({ outcome: 'plan_ready', capture: { runId: first.capture.runId } }); expect(record.state).toBeUndefined(); expect(fs.readFileSync(path.join(final.artifactDir!, 'terminal.raw.log'), 'utf8')).toBe('final output'); expect(fs.readFileSync(path.join(final.artifactDir!, 'terminal.visible.log'), 'utf8')).toBe(input.visible); const retry = persistPlanCountSnapshot(input, env); const retried = JSON.parse(fs.readFileSync(path.join(retry.artifactDir!, 'observation.json'), 'utf8')); expect(retry.artifactDir).not.toBe(final.artifactDir); expect(retried.capture.runId).not.toBe(first.capture.runId); expect(path.dirname(retry.artifactDir!)).not.toBe(path.dirname(final.artifactDir!)); } finally { fs.rmSync(root, { recursive: true, force: true }); } }); test('does not persist ordinary free runs without either capture setting', () => { expect(persistPlanCountSnapshot(input, {})).toEqual({}); expect(persistPlanCountSnapshot(input, { EVALS_RUN_ID: '', GSTACK_EVAL_DIR: '' })).toEqual({}); }); test('keeps run and skill identifiers within the owned artifact directory', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-')); try { const saved = persistPlanCountSnapshot({ ...input, skillName: '../../other' }, { EVALS_RUN_ID: '../outside', GSTACK_EVAL_DIR: root, }); expect(saved.artifactError).toBeUndefined(); expect(path.relative(root, saved.artifactDir!)).not.toStartWith('..'); } finally { fs.rmSync(root, { recursive: true, force: true }); } }); test('reports a diagnostic write failure without replacing the original observation', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-')); const file = path.join(root, 'file'); fs.writeFileSync(file, 'unchanged'); try { const result = persistPlanCountSnapshot(input, { EVALS_RUN_ID: 'run', GSTACK_EVAL_DIR: file }); expect(result.artifactError).toBeDefined(); expect(input.observation).toEqual({ outcome: 'timeout', reviewCount: 3 }); expect(fs.readFileSync(file, 'utf8')).toBe('unchanged'); } finally { fs.rmSync(root, { recursive: true, force: true }); } }); });