mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-09 14:38:59 +02:00
feat: stamp the claude CLI version into every eval-store run record
Three harness breakages were traced to claude-CLI TUI drift only after long flake hunts, because no run record said which CLI it actually exercised. EvalCollector now stamps claude_cli_version (claude --version, cached once per process, 'unknown' when the binary is absent) into both partial and finalized records — schema-additive optional field, no SCHEMA_VERSION bump. Correlating a flake wave with a CLI release becomes a grep over ~/.gstack/projects/<slug>/evals/ instead of archaeology. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
be3e9f0bfe
commit
0bbce3ea58
@@ -114,6 +114,11 @@ describe('EvalCollector', () => {
|
||||
expect(data.total_duration_ms).toBe(3000);
|
||||
expect(data.timestamp).toBeTruthy();
|
||||
expect(data.hostname).toBeTruthy();
|
||||
// CLI version stamping: always a non-empty string ('unknown' when the
|
||||
// claude binary is absent — the field must exist either way so flake
|
||||
// investigations can correlate runs with the TUI they exercised).
|
||||
expect(typeof data.claude_cli_version).toBe('string');
|
||||
expect(data.claude_cli_version!.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('finalize creates directory if missing', async () => {
|
||||
|
||||
@@ -118,6 +118,10 @@ export interface EvalResult {
|
||||
git_sha: string;
|
||||
timestamp: string;
|
||||
hostname: string;
|
||||
/** `claude --version` first line at run time (schema-additive, optional).
|
||||
* TUI drift broke the PTY harness three times before runs recorded which
|
||||
* CLI they actually exercised. */
|
||||
claude_cli_version?: string;
|
||||
tier: 'e2e' | 'llm-judge';
|
||||
total_tests: number;
|
||||
passed: number;
|
||||
@@ -777,6 +781,23 @@ function getVersion(): string {
|
||||
}
|
||||
}
|
||||
|
||||
// Cached per process: savePartial runs after EVERY test and must not pay a
|
||||
// CLI spawn each time. Three separate harness breakages were traced to
|
||||
// claude-CLI TUI drift only after long flake hunts — stamping the version
|
||||
// into every run record makes that correlation a grep instead of an
|
||||
// archaeology dig.
|
||||
let claudeCliVersionCache: string | null = null;
|
||||
export function getClaudeCliVersion(): string {
|
||||
if (claudeCliVersionCache !== null) return claudeCliVersionCache;
|
||||
try {
|
||||
const result = spawnSync('claude', ['--version'], { stdio: 'pipe', timeout: 10_000 });
|
||||
claudeCliVersionCache = result.stdout?.toString().split('\n')[0].trim() || 'unknown';
|
||||
} catch {
|
||||
claudeCliVersionCache = 'unknown';
|
||||
}
|
||||
return claudeCliVersionCache;
|
||||
}
|
||||
|
||||
export class EvalCollector {
|
||||
private tier: 'e2e' | 'llm-judge';
|
||||
private tests: EvalTestEntry[] = [];
|
||||
@@ -812,6 +833,7 @@ export class EvalCollector {
|
||||
git_sha: git.sha,
|
||||
timestamp: new Date().toISOString(),
|
||||
hostname: os.hostname(),
|
||||
claude_cli_version: getClaudeCliVersion(),
|
||||
tier: this.tier,
|
||||
total_tests: this.tests.length,
|
||||
passed,
|
||||
@@ -849,6 +871,7 @@ export class EvalCollector {
|
||||
git_sha: git.sha,
|
||||
timestamp,
|
||||
hostname: os.hostname(),
|
||||
claude_cli_version: getClaudeCliVersion(),
|
||||
tier: this.tier,
|
||||
total_tests: this.tests.length,
|
||||
passed,
|
||||
|
||||
Reference in New Issue
Block a user