fix(evals): cut path variance at its measured sources

- gstack-qa-evidence capture prints startedAt/completedAt/durationMs and, for
  --deadline captures, remainingMs; the functional report takes durations from
  them. The section clock notice asks for one clock read up front instead of one
  after every checkpoint (QA runs spent 7-14% of tool calls on date -u).
- ship plan-completion: skip the audit dispatch when discovery already found no
  plan (the dispatch-vs-skip conflict produced an optional 60-100 s subagent).
- materialize/checkpoint validation errors state the expected schema, so a
  rejected annotations file is fixable in one call instead of blocking the phase.
- session-runner counts turns from the transcript when a run times out, so
  timeouts stop reporting 'turn 0'.
This commit is contained in:
garrytan committed 2026-09-30 15:59:58 +00:00
1 parent a8a32e3216
commit 79092b22ea
8 files changed
+36 -22

No files matched your search

+10 -1
View File
@@ -35,6 +35,8 @@ test('native capture executes once, preserves exact JSON and stderr, and materia
expect(result.status, result.stderr).toBe(0);
const captured = receipt(result.stdout);
expect(captured).toMatchObject({ action: 'capture', status: 'complete', id: '001', exitCode: 0 });
expect(captured.durationMs).toBe(Date.parse(captured.completedAt) - Date.parse(captured.startedAt));
expect(captured.remainingMs).toBeUndefined();
expect(result.stdout).not.toContain('stateRoot');
expect(result.stderr).toBe('');
expect(fs.readFileSync(path.join(f.root, 'effects'), 'utf8')).toBe('once');
@@ -47,6 +49,10 @@ test('native capture executes once, preserves exact JSON and stderr, and materia
expect(JSON.parse(fs.readFileSync(path.join(f.root, 'exploration-001.json'), 'utf8'))).toEqual({
observationCommand: 'first native command', observed, hypothesis: 'The successful boundary suggests testing the rejected input next.', nextCommand: 'second native command',
});
f.json('annotations.json', { revision: 'revision', runtime: 'runtime', cwd: f.root, evidence: [], learning: [] });
const rejected = f.run('materialize', f.root, 'annotations.json');
expect(rejected.status).toBe(2);
expect(receipt(rejected.stderr).message).toContain('limits (non-empty string array)');
f.json('annotations.json', { revision: 'revision', runtime: 'runtime', cwd: f.root, limits: ['Only the declared contract was checked.'], evidence: [{ capture: '001', command: 'first native command', contract: 'README.md', expected: 'Declared exact result', classification: 'pass' }], learning: ['001'] });
const report = f.run('materialize', f.root, 'annotations.json');
expect(report.status, report.stderr).toBe(0);
@@ -94,7 +100,10 @@ test('capture shares a working-directory-relative deadline without resetting or
const before = fs.readFileSync(path.join(f.root, 'reports/deadline.json'));
const result = f.run('capture', 'reports', '001', '--deadline', 'reports/deadline.json', '--', process.execPath, '-e', 'console.log("{}")');
expect(result.status, result.stderr).toBe(0);
expect(receipt(result.stdout)).toMatchObject({ status: 'complete', exitCode: 0 });
const captured = receipt(result.stdout);
expect(captured).toMatchObject({ status: 'complete', exitCode: 0 });
expect(captured.remainingMs).toBeGreaterThan(0);
expect(captured.remainingMs).toBeLessThanOrEqual(5000);
expect(fs.readFileSync(path.join(f.root, 'reports/deadline.json'))).toEqual(before);
expect(fs.existsSync(path.join(f.root, 'reports/.qa-evidence/001/deadline.json'))).toBe(false);
});