mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 01:46:55 +02:00
fix(evals): cut path variance at its measured sources
- gstack-qa-evidence capture prints startedAt/completedAt/durationMs and, for --deadline captures, remainingMs; the functional report takes durations from them. The section clock notice asks for one clock read up front instead of one after every checkpoint (QA runs spent 7-14% of tool calls on date -u). - ship plan-completion: skip the audit dispatch when discovery already found no plan (the dispatch-vs-skip conflict produced an optional 60-100 s subagent). - materialize/checkpoint validation errors state the expected schema, so a rejected annotations file is fixable in one call instead of blocking the phase. - session-runner counts turns from the transcript when a run times out, so timeouts stop reporting 'turn 0'.
This commit is contained in:
1 parent
a8a32e3216
commit
79092b22ea
8 files changed
+36
-22
No files matched your search
+3
-3
@@ -1556,9 +1556,9 @@ The child reads the plan and every referenced
|
||||
code file; the parent validates its report and applies the gates below.
|
||||
|
||||
**Subagent prompt:** Substitute `<base>` and supply the active plan's absolute path
|
||||
or complete text, including relevant user-approved scope changes. If none exists,
|
||||
say so explicitly and let the child use the fallback search below. The child does
|
||||
not inherit the parent's conversation.
|
||||
or complete text, including user-approved scope changes. If none is known, say
|
||||
so; the child runs the fallback search below. If discovery found no plan, skip
|
||||
dispatch. The child does not inherit the parent's conversation.
|
||||
|
||||
````text
|
||||
You are running a ship-workflow plan completion audit. The base branch is `<base>`. Use `git diff origin/<base>` and inspect untracked files from `git status` to see the full proposed change. Do not commit or push. Report only: classify every item, but do not execute Gate Logic, ask the user, or advance the workflow. The parent applies those gates to your report.
|
||||
|
||||
+3
-3
@@ -1536,9 +1536,9 @@ The child reads the plan and every referenced
|
||||
code file; the parent validates its report and applies the gates below.
|
||||
|
||||
**Subagent prompt:** Substitute `<base>` and supply the active plan's absolute path
|
||||
or complete text, including relevant user-approved scope changes. If none exists,
|
||||
say so explicitly and let the child use the fallback search below. The child does
|
||||
not inherit the parent's conversation.
|
||||
or complete text, including user-approved scope changes. If none is known, say
|
||||
so; the child runs the fallback search below. If discovery found no plan, skip
|
||||
dispatch. The child does not inherit the parent's conversation.
|
||||
|
||||
````text
|
||||
You are running a ship-workflow plan completion audit. The base branch is `<base>`. Use `git diff origin/<base>` and inspect untracked files from `git status` to see the full proposed change. Do not commit or push. Report only: classify every item, but do not execute Gate Logic, ask the user, or advance the workflow. The parent applies those gates to your report.
|
||||
|
||||
@@ -293,7 +293,7 @@ Runner entry UTC: ${new Date(startTime).toISOString()}
|
||||
Hard deadline UTC: ${new Date(deadline).toISOString()}
|
||||
Completion reserve starts UTC: ${new Date(deadline - reserve).toISOString()}
|
||||
Setup, CLI startup and API queueing consume this same window; it never resets.
|
||||
Before source Reads and after each saved checkpoint, use Bash to run exactly \`date -u +%Y-%m-%dT%H:%M:%SZ\`. Compare that observed UTC time with the times above. When remaining time is at most ${reserve / 1000} seconds, prioritize the remaining required completion outputs and verification. No required content or gate may be skipped. If the clock read fails, report timing unavailable; do not invent remaining time or restart the deadline.`;
|
||||
Before source Reads, use Bash to run exactly \`date -u +%Y-%m-%dT%H:%M:%SZ\`. After each saved checkpoint, compare the latest evidence capture's printed completedAt with the times above; run that clock read again only when no capture has completed since your last clock read. When remaining time is at most ${reserve / 1000} seconds, prioritize the remaining required completion outputs and verification. No required content or gate may be skipped. If the clock read fails, report timing unavailable; do not invent remaining time or restart the deadline.`;
|
||||
systemPrompt = systemPrompt ? `${systemPrompt}\n\n${notice}` : notice;
|
||||
}
|
||||
|
||||
@@ -696,7 +696,8 @@ Before source Reads and after each saved checkpoint, use Bash to run exactly \`d
|
||||
}
|
||||
|
||||
// Cost from result line (exact) or estimate from chars
|
||||
const turnsUsed = resultLine?.num_turns || 0;
|
||||
const turnsUsed = resultLine?.num_turns
|
||||
|| new Set(transcript.filter(event => event.type === 'assistant' && !event.parent_tool_use_id).map(event => event.message?.id)).size;
|
||||
const estimatedCost = resultLine?.total_cost_usd || 0;
|
||||
const inputChars = prompt.length;
|
||||
const outputChars = (resultLine?.result || '').length;
|
||||
|
||||
@@ -35,6 +35,8 @@ test('native capture executes once, preserves exact JSON and stderr, and materia
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
const captured = receipt(result.stdout);
|
||||
expect(captured).toMatchObject({ action: 'capture', status: 'complete', id: '001', exitCode: 0 });
|
||||
expect(captured.durationMs).toBe(Date.parse(captured.completedAt) - Date.parse(captured.startedAt));
|
||||
expect(captured.remainingMs).toBeUndefined();
|
||||
expect(result.stdout).not.toContain('stateRoot');
|
||||
expect(result.stderr).toBe('');
|
||||
expect(fs.readFileSync(path.join(f.root, 'effects'), 'utf8')).toBe('once');
|
||||
@@ -47,6 +49,10 @@ test('native capture executes once, preserves exact JSON and stderr, and materia
|
||||
expect(JSON.parse(fs.readFileSync(path.join(f.root, 'exploration-001.json'), 'utf8'))).toEqual({
|
||||
observationCommand: 'first native command', observed, hypothesis: 'The successful boundary suggests testing the rejected input next.', nextCommand: 'second native command',
|
||||
});
|
||||
f.json('annotations.json', { revision: 'revision', runtime: 'runtime', cwd: f.root, evidence: [], learning: [] });
|
||||
const rejected = f.run('materialize', f.root, 'annotations.json');
|
||||
expect(rejected.status).toBe(2);
|
||||
expect(receipt(rejected.stderr).message).toContain('limits (non-empty string array)');
|
||||
f.json('annotations.json', { revision: 'revision', runtime: 'runtime', cwd: f.root, limits: ['Only the declared contract was checked.'], evidence: [{ capture: '001', command: 'first native command', contract: 'README.md', expected: 'Declared exact result', classification: 'pass' }], learning: ['001'] });
|
||||
const report = f.run('materialize', f.root, 'annotations.json');
|
||||
expect(report.status, report.stderr).toBe(0);
|
||||
@@ -94,7 +100,10 @@ test('capture shares a working-directory-relative deadline without resetting or
|
||||
const before = fs.readFileSync(path.join(f.root, 'reports/deadline.json'));
|
||||
const result = f.run('capture', 'reports', '001', '--deadline', 'reports/deadline.json', '--', process.execPath, '-e', 'console.log("{}")');
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
expect(receipt(result.stdout)).toMatchObject({ status: 'complete', exitCode: 0 });
|
||||
const captured = receipt(result.stdout);
|
||||
expect(captured).toMatchObject({ status: 'complete', exitCode: 0 });
|
||||
expect(captured.remainingMs).toBeGreaterThan(0);
|
||||
expect(captured.remainingMs).toBeLessThanOrEqual(5000);
|
||||
expect(fs.readFileSync(path.join(f.root, 'reports/deadline.json'))).toEqual(before);
|
||||
expect(fs.existsSync(path.join(f.root, 'reports/.qa-evidence/001/deadline.json'))).toBe(false);
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user