mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-02 17:40:02 +02:00
garrytan
f1eeec384e
Replaying run 36385945043's two Design attempts showed the existing routes
rejected correct endings: attempt 1 at the typed-completion path field
('- Reviewed plan written to …' is not a 'Plan written to' line), attempt 2
at the leading-fence veto (its final message opens with the dashboard).
nativePlanTerminalPreconditions is the structural prefix of
hasNativePlanTerminal (behavior unchanged). structuredPlanCompletion adds,
inside the existing nativeSummary branch: a complete report (Design
binding for Design), a completed review-log row for the expected skill
appended during this attempt under the child's GSTACK_HOME/project slug
(resolved with bin/gstack-slug) and stamped with the fixture commit, timed
between the report/last answer (second resolution) and the final native
message, a final message with stop_reason end_turn (now carried on public
transcript messages), and no visible question or permission prompt.
Timeout summaries add idleFor and lastTerminalCandidate. Terminal and throw
captures copy the plan file and review-log rows into the artifact
directory; copies are best-effort and recorded in evidence-copy.json.
Free regressions: both captured Design endings (trimmed fixture with
provenance; report, row and end_turn reconstructed and labelled), the
negative controls, and real-PTY completion/timeout runs through the real
review logger.
159 lines
10 KiB
TypeScript
159 lines
10 KiB
TypeScript
import { describe, expect, test } from 'bun:test';
|
|
import * as fs from 'node:fs';
|
|
import * as os from 'node:os';
|
|
import * as path from 'node:path';
|
|
import { copyPlanCountEvidence, createPlanCountSnapshotWriter, persistPlanCountSnapshot } from './helpers/plan-count-artifacts';
|
|
|
|
const input = {
|
|
skillName: 'plan-design-review', observation: { outcome: 'timeout', reviewCount: 3 },
|
|
raw: '\x1b[2JSTART\r' + 'raw-frame\r'.repeat(2000) + '\x1b[31mEND',
|
|
visible: 'START\r' + 'visible-frame\r'.repeat(2000) + 'END',
|
|
cwd: '/temporary/count-fixture', claudeConfigDir: '/temporary/claude-config',
|
|
};
|
|
|
|
describe('plan-count diagnostic artifacts', () => {
|
|
test('refreshes one in-progress attempt through final outcome without keeping old copies', () => {
|
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-'));
|
|
try {
|
|
const save = createPlanCountSnapshotWriter({ EVALS_RUN_ID: 'checkpoint', GSTACK_EVAL_DIR: root });
|
|
const progress = save({ ...input, observation: { state: 'in_progress', reviewCount: 1 } });
|
|
const recordPath = path.join(progress.artifactDir!, 'observation.json');
|
|
expect(JSON.parse(fs.readFileSync(recordPath, 'utf8'))).toMatchObject({ state: 'in_progress', reviewCount: 1 });
|
|
const final = save({ ...input, raw: input.raw + '\nlast question', observation: { outcome: 'completion_summary', reviewCount: 5 } });
|
|
expect(final.artifactDir).toBe(progress.artifactDir);
|
|
const record = JSON.parse(fs.readFileSync(recordPath, 'utf8'));
|
|
expect(record).toMatchObject({ outcome: 'completion_summary', reviewCount: 5 });
|
|
expect(record.state).toBeUndefined();
|
|
expect(fs.readdirSync(final.artifactDir!).sort()).toEqual(['observation.json', 'terminal.raw.log', 'terminal.visible.log']);
|
|
expect(fs.readFileSync(path.join(final.artifactDir!, 'terminal.raw.log'), 'utf8')).toBe(input.raw + '\nlast question');
|
|
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
|
});
|
|
test('keeps complete output and distinct retry records in the configured eval directory', () => {
|
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-'));
|
|
try {
|
|
const env = { EVALS_RUN_ID: 'same-run', GSTACK_EVAL_DIR: root };
|
|
const first = persistPlanCountSnapshot(input, env);
|
|
const second = persistPlanCountSnapshot({ ...input, raw: 'retry' }, env);
|
|
expect(first.artifactError).toBeUndefined();
|
|
expect(second.artifactError).toBeUndefined();
|
|
expect(first.artifactDir).not.toBe(second.artifactDir);
|
|
expect(path.relative(root, first.artifactDir!)).toStartWith(path.join('pty-count', 'same-run'));
|
|
expect(fs.readFileSync(path.join(first.artifactDir!, 'terminal.raw.log'), 'utf8')).toBe(input.raw);
|
|
expect(fs.readFileSync(path.join(first.artifactDir!, 'terminal.visible.log'), 'utf8')).toBe(input.visible);
|
|
const record = JSON.parse(fs.readFileSync(path.join(first.artifactDir!, 'observation.json'), 'utf8'));
|
|
expect(record).toMatchObject({ ...input.observation, artifactDir: first.artifactDir,
|
|
capture: { skill: input.skillName, cwd: input.cwd, claudeConfigDir: input.claudeConfigDir } });
|
|
expect(fs.readFileSync(path.join(second.artifactDir!, 'terminal.raw.log'), 'utf8')).toBe('retry');
|
|
if (process.platform !== 'win32') {
|
|
expect(fs.statSync(path.join(first.artifactDir!, 'terminal.raw.log')).mode & 0o777).toBe(0o600);
|
|
}
|
|
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('retains explicitly requested captures without a run ID and isolates invocation fallbacks', () => {
|
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-'));
|
|
try {
|
|
// The paid shard launcher sets GSTACK_EVAL_DIR without EVALS_RUN_ID.
|
|
const env = { GSTACK_EVAL_DIR: root };
|
|
const save = createPlanCountSnapshotWriter(env);
|
|
const progress = save({ ...input, observation: { state: 'in_progress', reviewCount: 1 } });
|
|
expect(progress.artifactError).toBeUndefined();
|
|
expect(progress.artifactDir).toBeDefined();
|
|
const first = JSON.parse(fs.readFileSync(path.join(progress.artifactDir!, 'observation.json'), 'utf8'));
|
|
expect(first.capture.runId).toMatch(/^local-[0-9a-f-]{36}$/);
|
|
expect(path.relative(root, progress.artifactDir!)).toStartWith(path.join('pty-count', first.capture.runId));
|
|
|
|
const final = save({ ...input, raw: 'final output', observation: { outcome: 'plan_ready', reviewCount: 5 } });
|
|
expect(final.artifactDir).toBe(progress.artifactDir);
|
|
const record = JSON.parse(fs.readFileSync(path.join(final.artifactDir!, 'observation.json'), 'utf8'));
|
|
expect(record).toMatchObject({ outcome: 'plan_ready', capture: { runId: first.capture.runId } });
|
|
expect(record.state).toBeUndefined();
|
|
expect(fs.readFileSync(path.join(final.artifactDir!, 'terminal.raw.log'), 'utf8')).toBe('final output');
|
|
expect(fs.readFileSync(path.join(final.artifactDir!, 'terminal.visible.log'), 'utf8')).toBe(input.visible);
|
|
|
|
const retry = persistPlanCountSnapshot(input, env);
|
|
const retried = JSON.parse(fs.readFileSync(path.join(retry.artifactDir!, 'observation.json'), 'utf8'));
|
|
expect(retry.artifactDir).not.toBe(final.artifactDir);
|
|
expect(retried.capture.runId).not.toBe(first.capture.runId);
|
|
expect(path.dirname(retry.artifactDir!)).not.toBe(path.dirname(final.artifactDir!));
|
|
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('does not persist ordinary free runs without either capture setting', () => {
|
|
expect(persistPlanCountSnapshot(input, {})).toEqual({});
|
|
expect(persistPlanCountSnapshot(input, { EVALS_RUN_ID: '', GSTACK_EVAL_DIR: '' })).toEqual({});
|
|
});
|
|
|
|
test('keeps run and skill identifiers within the owned artifact directory', () => {
|
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-'));
|
|
try {
|
|
const saved = persistPlanCountSnapshot({ ...input, skillName: '../../other' }, {
|
|
EVALS_RUN_ID: '../outside', GSTACK_EVAL_DIR: root,
|
|
});
|
|
expect(saved.artifactError).toBeUndefined();
|
|
expect(path.relative(root, saved.artifactDir!)).not.toStartWith('..');
|
|
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('reports a diagnostic write failure without replacing the original observation', () => {
|
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-'));
|
|
const file = path.join(root, 'file');
|
|
fs.writeFileSync(file, 'unchanged');
|
|
try {
|
|
const result = persistPlanCountSnapshot(input, { EVALS_RUN_ID: 'run', GSTACK_EVAL_DIR: file });
|
|
expect(result.artifactError).toBeDefined();
|
|
expect(input.observation).toEqual({ outcome: 'timeout', reviewCount: 3 });
|
|
expect(fs.readFileSync(file, 'utf8')).toBe('unchanged');
|
|
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
|
});
|
|
test('copies the plan and review-log rows so they survive fixture cleanup', () => {
|
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-'));
|
|
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-fixture-'));
|
|
try {
|
|
const planPath = path.join(fixture, 'plan.md');
|
|
const logs = path.join(fixture, 'state', 'projects', 'slug');
|
|
fs.mkdirSync(logs, { recursive: true });
|
|
fs.writeFileSync(planPath, '# plan\n## GSTACK REVIEW REPORT\n');
|
|
fs.writeFileSync(path.join(logs, 'main-reviews.jsonl'), '{"skill":"plan-design-review","status":"clean"}\n');
|
|
fs.writeFileSync(path.join(logs, 'unrelated.json'), '{}');
|
|
const result = persistPlanCountSnapshot(input, { EVALS_RUN_ID: 'run', GSTACK_EVAL_DIR: root });
|
|
copyPlanCountEvidence(result.artifactDir, { planPath, reviewLogDirectory: logs });
|
|
fs.rmSync(fixture, { recursive: true, force: true });
|
|
expect(result.artifactError).toBeUndefined();
|
|
expect(fs.readFileSync(path.join(result.artifactDir!, 'evidence', 'plan', 'plan.md'), 'utf8')).toBe('# plan\n## GSTACK REVIEW REPORT\n');
|
|
expect(fs.readFileSync(path.join(result.artifactDir!, 'evidence', 'review-log', 'main-reviews.jsonl'), 'utf8'))
|
|
.toBe('{"skill":"plan-design-review","status":"clean"}\n');
|
|
expect(fs.existsSync(path.join(result.artifactDir!, 'evidence', 'review-log', 'unrelated.json'))).toBe(false);
|
|
expect(JSON.parse(fs.readFileSync(path.join(result.artifactDir!, 'evidence-copy.json'), 'utf8'))).toEqual({
|
|
plan: `copied: ${planPath}`, 'reviewLog:main-reviews.jsonl': `copied: ${path.join(logs, 'main-reviews.jsonl')}`,
|
|
});
|
|
expect(JSON.parse(fs.readFileSync(path.join(result.artifactDir!, 'observation.json'), 'utf8')).outcome).toBe('timeout');
|
|
} finally {
|
|
fs.rmSync(root, { recursive: true, force: true });
|
|
fs.rmSync(fixture, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test('a missing plan or unreadable log is recorded and never replaces the observation', () => {
|
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-'));
|
|
try {
|
|
const planPath = path.join(root, 'never-written.md');
|
|
const notDirectory = path.join(root, 'not-a-directory');
|
|
fs.writeFileSync(notDirectory, 'file');
|
|
const result = persistPlanCountSnapshot(input, { EVALS_RUN_ID: 'run', GSTACK_EVAL_DIR: path.join(root, 'evals') });
|
|
expect(() => copyPlanCountEvidence(result.artifactDir, { planPath, reviewLogDirectory: notDirectory })).not.toThrow();
|
|
expect(result.artifactError).toBeUndefined();
|
|
const copy = JSON.parse(fs.readFileSync(path.join(result.artifactDir!, 'evidence-copy.json'), 'utf8'));
|
|
expect(copy.plan).toBe(`missing: ${planPath}`);
|
|
expect(copy.reviewLog).toStartWith('error: ');
|
|
expect(JSON.parse(fs.readFileSync(path.join(result.artifactDir!, 'observation.json'), 'utf8')))
|
|
.toMatchObject({ outcome: 'timeout', reviewCount: 3 });
|
|
const bare = persistPlanCountSnapshot(input, { EVALS_RUN_ID: 'run', GSTACK_EVAL_DIR: path.join(root, 'evals') });
|
|
copyPlanCountEvidence(bare.artifactDir, {});
|
|
expect(() => copyPlanCountEvidence(undefined, { planPath })).not.toThrow();
|
|
expect(JSON.parse(fs.readFileSync(path.join(bare.artifactDir!, 'evidence-copy.json'), 'utf8'))).toEqual({
|
|
plan: 'missing: no expected plan path', reviewLog: 'missing: no fixture-owned review log binding' });
|
|
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
|
});
|
|
});
|