mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 01:46:55 +02:00
test: judge plan-count completion on structured evidence, not wording
Replaying run 36385945043's two Design attempts showed the existing routes
rejected correct endings: attempt 1 at the typed-completion path field
('- Reviewed plan written to …' is not a 'Plan written to' line), attempt 2
at the leading-fence veto (its final message opens with the dashboard).
nativePlanTerminalPreconditions is the structural prefix of
hasNativePlanTerminal (behavior unchanged). structuredPlanCompletion adds,
inside the existing nativeSummary branch: a complete report (Design
binding for Design), a completed review-log row for the expected skill
appended during this attempt under the child's GSTACK_HOME/project slug
(resolved with bin/gstack-slug) and stamped with the fixture commit, timed
between the report/last answer (second resolution) and the final native
message, a final message with stop_reason end_turn (now carried on public
transcript messages), and no visible question or permission prompt.
Timeout summaries add idleFor and lastTerminalCandidate. Terminal and throw
captures copy the plan file and review-log rows into the artifact
directory; copies are best-effort and recorded in evidence-copy.json.
Free regressions: both captured Design endings (trimmed fixture with
provenance; report, row and end_turn reconstructed and labelled), the
negative controls, and real-PTY completion/timeout runs through the real
review logger.
This commit is contained in:
1 parent
a0aa1e8f39
commit
f1eeec384e
6 files changed
+1172
-32
No files matched your search
@@ -13,6 +13,37 @@ interface PlanCountSnapshot {
|
||||
claudeConfigDir: string | null;
|
||||
}
|
||||
|
||||
/** Copy the caller-owned plan and review-log rows into an attempt's artifact
|
||||
* directory before fixture cleanup removes them. Best-effort: each result
|
||||
* (copied, missing or its error) is recorded in evidence-copy.json and never
|
||||
* replaces the observation or its outcome. */
|
||||
export function copyPlanCountEvidence(artifactDir: string | undefined,
|
||||
evidence: { planPath?: string; reviewLogDirectory?: string }): void {
|
||||
if (!artifactDir) return;
|
||||
const results: Record<string, string> = {};
|
||||
const copy = (label: string, source: string, target: string) => {
|
||||
try {
|
||||
if (!fs.existsSync(source)) { results[label] = `missing: ${source}`; return; }
|
||||
fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 });
|
||||
fs.copyFileSync(source, target);
|
||||
fs.chmodSync(target, 0o600);
|
||||
results[label] = `copied: ${source}`;
|
||||
} catch (error) { results[label] = `error: ${String(error)}`; }
|
||||
};
|
||||
if (evidence.planPath) copy('plan', evidence.planPath, path.join(artifactDir, 'evidence', 'plan', path.basename(evidence.planPath)));
|
||||
else results.plan = 'missing: no expected plan path';
|
||||
if (evidence.reviewLogDirectory) {
|
||||
let names: string[] = [];
|
||||
try { names = fs.readdirSync(evidence.reviewLogDirectory).filter(name => name.endsWith('-reviews.jsonl')); }
|
||||
catch (error) { results.reviewLog = fs.existsSync(evidence.reviewLogDirectory) ? `error: ${String(error)}` : `missing: ${evidence.reviewLogDirectory}`; }
|
||||
if (!names.length && !results.reviewLog) results.reviewLog = `missing: no *-reviews.jsonl in ${evidence.reviewLogDirectory}`;
|
||||
for (const name of names) copy(`reviewLog:${name}`, path.join(evidence.reviewLogDirectory, name), path.join(artifactDir, 'evidence', 'review-log', name));
|
||||
} else results.reviewLog = 'missing: no fixture-owned review log binding';
|
||||
try {
|
||||
fs.writeFileSync(path.join(artifactDir, 'evidence-copy.json'), JSON.stringify(results, null, 2) + '\n', { mode: 0o600 });
|
||||
} catch { /* the observation and its outcome stay authoritative */ }
|
||||
}
|
||||
|
||||
/** One owned directory per count attempt; periodic captures replace files atomically. */
|
||||
export function createPlanCountSnapshotWriter(env: NodeJS.ProcessEnv = process.env):
|
||||
(input: PlanCountSnapshot) => { artifactDir?: string; artifactError?: string } {
|
||||
|
||||
Reference in new issue
Block a user