feat: stamp the claude CLI version into every eval-store run record

Three harness breakages were traced to claude-CLI TUI drift only after long
flake hunts, because no run record said which CLI it actually exercised.
EvalCollector now stamps claude_cli_version (claude --version, cached once
per process, 'unknown' when the binary is absent) into both partial and
finalized records — schema-additive optional field, no SCHEMA_VERSION bump.
Correlating a flake wave with a CLI release becomes a grep over
~/.gstack/projects/<slug>/evals/ instead of archaeology.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-31 04:06:06 +00:00
co-authored by Claude Fable 5
parent be3e9f0bfe
commit 0bbce3ea58
2 changed files with 28 additions and 0 deletions
+5
View File
@@ -114,6 +114,11 @@ describe('EvalCollector', () => {
expect(data.total_duration_ms).toBe(3000);
expect(data.timestamp).toBeTruthy();
expect(data.hostname).toBeTruthy();
// CLI version stamping: always a non-empty string ('unknown' when the
// claude binary is absent — the field must exist either way so flake
// investigations can correlate runs with the TUI they exercised).
expect(typeof data.claude_cli_version).toBe('string');
expect(data.claude_cli_version!.length).toBeGreaterThan(0);
});
test('finalize creates directory if missing', async () => {
+23
View File
@@ -118,6 +118,10 @@ export interface EvalResult {
git_sha: string;
timestamp: string;
hostname: string;
/** `claude --version` first line at run time (schema-additive, optional).
* TUI drift broke the PTY harness three times before runs recorded which
* CLI they actually exercised. */
claude_cli_version?: string;
tier: 'e2e' | 'llm-judge';
total_tests: number;
passed: number;
@@ -777,6 +781,23 @@ function getVersion(): string {
}
}
// Cached per process: savePartial runs after EVERY test and must not pay a
// CLI spawn each time. Three separate harness breakages were traced to
// claude-CLI TUI drift only after long flake hunts — stamping the version
// into every run record makes that correlation a grep instead of an
// archaeology dig.
let claudeCliVersionCache: string | null = null;
export function getClaudeCliVersion(): string {
if (claudeCliVersionCache !== null) return claudeCliVersionCache;
try {
const result = spawnSync('claude', ['--version'], { stdio: 'pipe', timeout: 10_000 });
claudeCliVersionCache = result.stdout?.toString().split('\n')[0].trim() || 'unknown';
} catch {
claudeCliVersionCache = 'unknown';
}
return claudeCliVersionCache;
}
export class EvalCollector {
private tier: 'e2e' | 'llm-judge';
private tests: EvalTestEntry[] = [];
@@ -812,6 +833,7 @@ export class EvalCollector {
git_sha: git.sha,
timestamp: new Date().toISOString(),
hostname: os.hostname(),
claude_cli_version: getClaudeCliVersion(),
tier: this.tier,
total_tests: this.tests.length,
passed,
@@ -849,6 +871,7 @@ export class EvalCollector {
git_sha: git.sha,
timestamp,
hostname: os.hostname(),
claude_cli_version: getClaudeCliVersion(),
tier: this.tier,
total_tests: this.tests.length,
passed,