test: judge plan-count completion on structured evidence, not wording

Replaying run 36385945043's two Design attempts showed the existing routes
rejected correct endings: attempt 1 at the typed-completion path field
('- Reviewed plan written to …' is not a 'Plan written to' line), attempt 2
at the leading-fence veto (its final message opens with the dashboard).

nativePlanTerminalPreconditions is the structural prefix of
hasNativePlanTerminal (behavior unchanged). structuredPlanCompletion adds,
inside the existing nativeSummary branch: a complete report (Design
binding for Design), a completed review-log row for the expected skill
appended during this attempt under the child's GSTACK_HOME/project slug
(resolved with bin/gstack-slug) and stamped with the fixture commit, timed
between the report/last answer (second resolution) and the final native
message, a final message with stop_reason end_turn (now carried on public
transcript messages), and no visible question or permission prompt.

Timeout summaries add idleFor and lastTerminalCandidate. Terminal and throw
captures copy the plan file and review-log rows into the artifact
directory; copies are best-effort and recorded in evidence-copy.json.
Free regressions: both captured Design endings (trimmed fixture with
provenance; report, row and end_turn reconstructed and labelled), the
negative controls, and real-PTY completion/timeout runs through the real
review logger.
This commit is contained in:
garrytan committed 2026-09-29 15:04:28 +00:00
1 parent a0aa1e8f39
commit f1eeec384e
6 files changed
+1172 -32

No files matched your search

+50 -1
View File
@@ -2,7 +2,7 @@ import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { createPlanCountSnapshotWriter, persistPlanCountSnapshot } from './helpers/plan-count-artifacts';
import { copyPlanCountEvidence, createPlanCountSnapshotWriter, persistPlanCountSnapshot } from './helpers/plan-count-artifacts';
const input = {
skillName: 'plan-design-review', observation: { outcome: 'timeout', reviewCount: 3 },
@@ -106,4 +106,53 @@ describe('plan-count diagnostic artifacts', () => {
expect(fs.readFileSync(file, 'utf8')).toBe('unchanged');
} finally { fs.rmSync(root, { recursive: true, force: true }); }
});
test('copies the plan and review-log rows so they survive fixture cleanup', () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-'));
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-fixture-'));
try {
const planPath = path.join(fixture, 'plan.md');
const logs = path.join(fixture, 'state', 'projects', 'slug');
fs.mkdirSync(logs, { recursive: true });
fs.writeFileSync(planPath, '# plan\n## GSTACK REVIEW REPORT\n');
fs.writeFileSync(path.join(logs, 'main-reviews.jsonl'), '{"skill":"plan-design-review","status":"clean"}\n');
fs.writeFileSync(path.join(logs, 'unrelated.json'), '{}');
const result = persistPlanCountSnapshot(input, { EVALS_RUN_ID: 'run', GSTACK_EVAL_DIR: root });
copyPlanCountEvidence(result.artifactDir, { planPath, reviewLogDirectory: logs });
fs.rmSync(fixture, { recursive: true, force: true });
expect(result.artifactError).toBeUndefined();
expect(fs.readFileSync(path.join(result.artifactDir!, 'evidence', 'plan', 'plan.md'), 'utf8')).toBe('# plan\n## GSTACK REVIEW REPORT\n');
expect(fs.readFileSync(path.join(result.artifactDir!, 'evidence', 'review-log', 'main-reviews.jsonl'), 'utf8'))
.toBe('{"skill":"plan-design-review","status":"clean"}\n');
expect(fs.existsSync(path.join(result.artifactDir!, 'evidence', 'review-log', 'unrelated.json'))).toBe(false);
expect(JSON.parse(fs.readFileSync(path.join(result.artifactDir!, 'evidence-copy.json'), 'utf8'))).toEqual({
plan: `copied: ${planPath}`, 'reviewLog:main-reviews.jsonl': `copied: ${path.join(logs, 'main-reviews.jsonl')}`,
});
expect(JSON.parse(fs.readFileSync(path.join(result.artifactDir!, 'observation.json'), 'utf8')).outcome).toBe('timeout');
} finally {
fs.rmSync(root, { recursive: true, force: true });
fs.rmSync(fixture, { recursive: true, force: true });
}
});
test('a missing plan or unreadable log is recorded and never replaces the observation', () => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-'));
try {
const planPath = path.join(root, 'never-written.md');
const notDirectory = path.join(root, 'not-a-directory');
fs.writeFileSync(notDirectory, 'file');
const result = persistPlanCountSnapshot(input, { EVALS_RUN_ID: 'run', GSTACK_EVAL_DIR: path.join(root, 'evals') });
expect(() => copyPlanCountEvidence(result.artifactDir, { planPath, reviewLogDirectory: notDirectory })).not.toThrow();
expect(result.artifactError).toBeUndefined();
const copy = JSON.parse(fs.readFileSync(path.join(result.artifactDir!, 'evidence-copy.json'), 'utf8'));
expect(copy.plan).toBe(`missing: ${planPath}`);
expect(copy.reviewLog).toStartWith('error: ');
expect(JSON.parse(fs.readFileSync(path.join(result.artifactDir!, 'observation.json'), 'utf8')))
.toMatchObject({ outcome: 'timeout', reviewCount: 3 });
const bare = persistPlanCountSnapshot(input, { EVALS_RUN_ID: 'run', GSTACK_EVAL_DIR: path.join(root, 'evals') });
copyPlanCountEvidence(bare.artifactDir, {});
expect(() => copyPlanCountEvidence(undefined, { planPath })).not.toThrow();
expect(JSON.parse(fs.readFileSync(path.join(bare.artifactDir!, 'evidence-copy.json'), 'utf8'))).toEqual({
plan: 'missing: no expected plan path', reviewLog: 'missing: no fixture-owned review log binding' });
} finally { fs.rmSync(root, { recursive: true, force: true }); }
});
});