test: judge plan-count completion on structured evidence, not wording

Replaying run 36385945043's two Design attempts showed the existing routes
rejected correct endings: attempt 1 at the typed-completion path field
('- Reviewed plan written to …' is not a 'Plan written to' line), attempt 2
at the leading-fence veto (its final message opens with the dashboard).

nativePlanTerminalPreconditions is the structural prefix of
hasNativePlanTerminal (behavior unchanged). structuredPlanCompletion adds,
inside the existing nativeSummary branch: a complete report (Design
binding for Design), a completed review-log row for the expected skill
appended during this attempt under the child's GSTACK_HOME/project slug
(resolved with bin/gstack-slug) and stamped with the fixture commit, timed
between the report/last answer (second resolution) and the final native
message, a final message with stop_reason end_turn (now carried on public
transcript messages), and no visible question or permission prompt.

Timeout summaries add idleFor and lastTerminalCandidate. Terminal and throw
captures copy the plan file and review-log rows into the artifact
directory; copies are best-effort and recorded in evidence-copy.json.
Free regressions: both captured Design endings (trimmed fixture with
provenance; report, row and end_turn reconstructed and labelled), the
negative controls, and real-PTY completion/timeout runs through the real
review logger.
This commit is contained in:
garrytan committed 2026-09-29 15:04:28 +00:00
1 parent a0aa1e8f39
commit f1eeec384e
6 files changed
+1172 -32

No files matched your search

+31
View File
@@ -13,6 +13,37 @@ interface PlanCountSnapshot {
claudeConfigDir: string | null;
}
/** Copy the caller-owned plan and review-log rows into an attempt's artifact
* directory before fixture cleanup removes them. Best-effort: each result
* (copied, missing or its error) is recorded in evidence-copy.json and never
* replaces the observation or its outcome. */
export function copyPlanCountEvidence(artifactDir: string | undefined,
evidence: { planPath?: string; reviewLogDirectory?: string }): void {
if (!artifactDir) return;
const results: Record<string, string> = {};
const copy = (label: string, source: string, target: string) => {
try {
if (!fs.existsSync(source)) { results[label] = `missing: ${source}`; return; }
fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 });
fs.copyFileSync(source, target);
fs.chmodSync(target, 0o600);
results[label] = `copied: ${source}`;
} catch (error) { results[label] = `error: ${String(error)}`; }
};
if (evidence.planPath) copy('plan', evidence.planPath, path.join(artifactDir, 'evidence', 'plan', path.basename(evidence.planPath)));
else results.plan = 'missing: no expected plan path';
if (evidence.reviewLogDirectory) {
let names: string[] = [];
try { names = fs.readdirSync(evidence.reviewLogDirectory).filter(name => name.endsWith('-reviews.jsonl')); }
catch (error) { results.reviewLog = fs.existsSync(evidence.reviewLogDirectory) ? `error: ${String(error)}` : `missing: ${evidence.reviewLogDirectory}`; }
if (!names.length && !results.reviewLog) results.reviewLog = `missing: no *-reviews.jsonl in ${evidence.reviewLogDirectory}`;
for (const name of names) copy(`reviewLog:${name}`, path.join(evidence.reviewLogDirectory, name), path.join(artifactDir, 'evidence', 'review-log', name));
} else results.reviewLog = 'missing: no fixture-owned review log binding';
try {
fs.writeFileSync(path.join(artifactDir, 'evidence-copy.json'), JSON.stringify(results, null, 2) + '\n', { mode: 0o600 });
} catch { /* the observation and its outcome stay authoritative */ }
}
/** One owned directory per count attempt; periodic captures replace files atomically. */
export function createPlanCountSnapshotWriter(env: NodeJS.ProcessEnv = process.env):
(input: PlanCountSnapshot) => { artifactDir?: string; artifactError?: string } {