mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 01:46:55 +02:00
test: judge plan-count completion on structured evidence, not wording
Replaying run 36385945043's two Design attempts showed the existing routes
rejected correct endings: attempt 1 at the typed-completion path field
('- Reviewed plan written to …' is not a 'Plan written to' line), attempt 2
at the leading-fence veto (its final message opens with the dashboard).
nativePlanTerminalPreconditions is the structural prefix of
hasNativePlanTerminal (behavior unchanged). structuredPlanCompletion adds,
inside the existing nativeSummary branch: a complete report (Design
binding for Design), a completed review-log row for the expected skill
appended during this attempt under the child's GSTACK_HOME/project slug
(resolved with bin/gstack-slug) and stamped with the fixture commit, timed
between the report/last answer (second resolution) and the final native
message, a final message with stop_reason end_turn (now carried on public
transcript messages), and no visible question or permission prompt.
Timeout summaries add idleFor and lastTerminalCandidate. Terminal and throw
captures copy the plan file and review-log rows into the artifact
directory; copies are best-effort and recorded in evidence-copy.json.
Free regressions: both captured Design endings (trimmed fixture with
provenance; report, row and end_turn reconstructed and labelled), the
negative controls, and real-PTY completion/timeout runs through the real
review logger.
This commit is contained in:
1 parent
a0aa1e8f39
commit
f1eeec384e
6 files changed
+1172
-32
No files matched your search
@@ -2,7 +2,7 @@ import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { createPlanCountSnapshotWriter, persistPlanCountSnapshot } from './helpers/plan-count-artifacts';
|
||||
import { copyPlanCountEvidence, createPlanCountSnapshotWriter, persistPlanCountSnapshot } from './helpers/plan-count-artifacts';
|
||||
|
||||
const input = {
|
||||
skillName: 'plan-design-review', observation: { outcome: 'timeout', reviewCount: 3 },
|
||||
@@ -106,4 +106,53 @@ describe('plan-count diagnostic artifacts', () => {
|
||||
expect(fs.readFileSync(file, 'utf8')).toBe('unchanged');
|
||||
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
test('copies the plan and review-log rows so they survive fixture cleanup', () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-'));
|
||||
const fixture = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-fixture-'));
|
||||
try {
|
||||
const planPath = path.join(fixture, 'plan.md');
|
||||
const logs = path.join(fixture, 'state', 'projects', 'slug');
|
||||
fs.mkdirSync(logs, { recursive: true });
|
||||
fs.writeFileSync(planPath, '# plan\n## GSTACK REVIEW REPORT\n');
|
||||
fs.writeFileSync(path.join(logs, 'main-reviews.jsonl'), '{"skill":"plan-design-review","status":"clean"}\n');
|
||||
fs.writeFileSync(path.join(logs, 'unrelated.json'), '{}');
|
||||
const result = persistPlanCountSnapshot(input, { EVALS_RUN_ID: 'run', GSTACK_EVAL_DIR: root });
|
||||
copyPlanCountEvidence(result.artifactDir, { planPath, reviewLogDirectory: logs });
|
||||
fs.rmSync(fixture, { recursive: true, force: true });
|
||||
expect(result.artifactError).toBeUndefined();
|
||||
expect(fs.readFileSync(path.join(result.artifactDir!, 'evidence', 'plan', 'plan.md'), 'utf8')).toBe('# plan\n## GSTACK REVIEW REPORT\n');
|
||||
expect(fs.readFileSync(path.join(result.artifactDir!, 'evidence', 'review-log', 'main-reviews.jsonl'), 'utf8'))
|
||||
.toBe('{"skill":"plan-design-review","status":"clean"}\n');
|
||||
expect(fs.existsSync(path.join(result.artifactDir!, 'evidence', 'review-log', 'unrelated.json'))).toBe(false);
|
||||
expect(JSON.parse(fs.readFileSync(path.join(result.artifactDir!, 'evidence-copy.json'), 'utf8'))).toEqual({
|
||||
plan: `copied: ${planPath}`, 'reviewLog:main-reviews.jsonl': `copied: ${path.join(logs, 'main-reviews.jsonl')}`,
|
||||
});
|
||||
expect(JSON.parse(fs.readFileSync(path.join(result.artifactDir!, 'observation.json'), 'utf8')).outcome).toBe('timeout');
|
||||
} finally {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
fs.rmSync(fixture, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('a missing plan or unreadable log is recorded and never replaces the observation', () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-count-artifacts-'));
|
||||
try {
|
||||
const planPath = path.join(root, 'never-written.md');
|
||||
const notDirectory = path.join(root, 'not-a-directory');
|
||||
fs.writeFileSync(notDirectory, 'file');
|
||||
const result = persistPlanCountSnapshot(input, { EVALS_RUN_ID: 'run', GSTACK_EVAL_DIR: path.join(root, 'evals') });
|
||||
expect(() => copyPlanCountEvidence(result.artifactDir, { planPath, reviewLogDirectory: notDirectory })).not.toThrow();
|
||||
expect(result.artifactError).toBeUndefined();
|
||||
const copy = JSON.parse(fs.readFileSync(path.join(result.artifactDir!, 'evidence-copy.json'), 'utf8'));
|
||||
expect(copy.plan).toBe(`missing: ${planPath}`);
|
||||
expect(copy.reviewLog).toStartWith('error: ');
|
||||
expect(JSON.parse(fs.readFileSync(path.join(result.artifactDir!, 'observation.json'), 'utf8')))
|
||||
.toMatchObject({ outcome: 'timeout', reviewCount: 3 });
|
||||
const bare = persistPlanCountSnapshot(input, { EVALS_RUN_ID: 'run', GSTACK_EVAL_DIR: path.join(root, 'evals') });
|
||||
copyPlanCountEvidence(bare.artifactDir, {});
|
||||
expect(() => copyPlanCountEvidence(undefined, { planPath })).not.toThrow();
|
||||
expect(JSON.parse(fs.readFileSync(path.join(bare.artifactDir!, 'evidence-copy.json'), 'utf8'))).toEqual({
|
||||
plan: 'missing: no expected plan path', reviewLog: 'missing: no fixture-owned review log binding' });
|
||||
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user