mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-15 09:25:28 +02:00
feat: stamp the claude CLI version into every eval-store run record
Three harness breakages were traced to claude-CLI TUI drift only after long flake hunts, because no run record said which CLI it actually exercised. EvalCollector now stamps claude_cli_version (claude --version, cached once per process, 'unknown' when the binary is absent) into both partial and finalized records — schema-additive optional field, no SCHEMA_VERSION bump. Correlating a flake wave with a CLI release becomes a grep over ~/.gstack/projects/<slug>/evals/ instead of archaeology. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
be3e9f0bfe
commit
0bbce3ea58
@@ -114,6 +114,11 @@ describe('EvalCollector', () => {
|
|||||||
expect(data.total_duration_ms).toBe(3000);
|
expect(data.total_duration_ms).toBe(3000);
|
||||||
expect(data.timestamp).toBeTruthy();
|
expect(data.timestamp).toBeTruthy();
|
||||||
expect(data.hostname).toBeTruthy();
|
expect(data.hostname).toBeTruthy();
|
||||||
|
// CLI version stamping: always a non-empty string ('unknown' when the
|
||||||
|
// claude binary is absent — the field must exist either way so flake
|
||||||
|
// investigations can correlate runs with the TUI they exercised).
|
||||||
|
expect(typeof data.claude_cli_version).toBe('string');
|
||||||
|
expect(data.claude_cli_version!.length).toBeGreaterThan(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('finalize creates directory if missing', async () => {
|
test('finalize creates directory if missing', async () => {
|
||||||
|
|||||||
@@ -118,6 +118,10 @@ export interface EvalResult {
|
|||||||
git_sha: string;
|
git_sha: string;
|
||||||
timestamp: string;
|
timestamp: string;
|
||||||
hostname: string;
|
hostname: string;
|
||||||
|
/** `claude --version` first line at run time (schema-additive, optional).
|
||||||
|
* TUI drift broke the PTY harness three times before runs recorded which
|
||||||
|
* CLI they actually exercised. */
|
||||||
|
claude_cli_version?: string;
|
||||||
tier: 'e2e' | 'llm-judge';
|
tier: 'e2e' | 'llm-judge';
|
||||||
total_tests: number;
|
total_tests: number;
|
||||||
passed: number;
|
passed: number;
|
||||||
@@ -777,6 +781,23 @@ function getVersion(): string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Cached per process: savePartial runs after EVERY test and must not pay a
|
||||||
|
// CLI spawn each time. Three separate harness breakages were traced to
|
||||||
|
// claude-CLI TUI drift only after long flake hunts — stamping the version
|
||||||
|
// into every run record makes that correlation a grep instead of an
|
||||||
|
// archaeology dig.
|
||||||
|
let claudeCliVersionCache: string | null = null;
|
||||||
|
export function getClaudeCliVersion(): string {
|
||||||
|
if (claudeCliVersionCache !== null) return claudeCliVersionCache;
|
||||||
|
try {
|
||||||
|
const result = spawnSync('claude', ['--version'], { stdio: 'pipe', timeout: 10_000 });
|
||||||
|
claudeCliVersionCache = result.stdout?.toString().split('\n')[0].trim() || 'unknown';
|
||||||
|
} catch {
|
||||||
|
claudeCliVersionCache = 'unknown';
|
||||||
|
}
|
||||||
|
return claudeCliVersionCache;
|
||||||
|
}
|
||||||
|
|
||||||
export class EvalCollector {
|
export class EvalCollector {
|
||||||
private tier: 'e2e' | 'llm-judge';
|
private tier: 'e2e' | 'llm-judge';
|
||||||
private tests: EvalTestEntry[] = [];
|
private tests: EvalTestEntry[] = [];
|
||||||
@@ -812,6 +833,7 @@ export class EvalCollector {
|
|||||||
git_sha: git.sha,
|
git_sha: git.sha,
|
||||||
timestamp: new Date().toISOString(),
|
timestamp: new Date().toISOString(),
|
||||||
hostname: os.hostname(),
|
hostname: os.hostname(),
|
||||||
|
claude_cli_version: getClaudeCliVersion(),
|
||||||
tier: this.tier,
|
tier: this.tier,
|
||||||
total_tests: this.tests.length,
|
total_tests: this.tests.length,
|
||||||
passed,
|
passed,
|
||||||
@@ -849,6 +871,7 @@ export class EvalCollector {
|
|||||||
git_sha: git.sha,
|
git_sha: git.sha,
|
||||||
timestamp,
|
timestamp,
|
||||||
hostname: os.hostname(),
|
hostname: os.hostname(),
|
||||||
|
claude_cli_version: getClaudeCliVersion(),
|
||||||
tier: this.tier,
|
tier: this.tier,
|
||||||
total_tests: this.tests.length,
|
total_tests: this.tests.length,
|
||||||
passed,
|
passed,
|
||||||
|
|||||||
Reference in New Issue
Block a user