CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-07-06 08:07:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: add E2E observability — heartbeat, progress.log, NDJSON persistence, savePartial()

Browse Source 
session-runner: atomic heartbeat file (e2e-live.json), per-run log directory
(~/.gstack-dev/e2e-runs/{runId}/), progress.log + per-test NDJSON persistence,
failure transcripts to persistent run dir instead of tmpdir.

eval-store: 3 new diagnostic fields (exit_reason, timeout_at_turn, last_tool_call),
savePartial() writes _partial-e2e.json after each addTest() for crash resilience,
finalize() cleans up partial file.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-03-14 11:04:16 -05:00
parent eb9a9193c9
commit f9cfabeda8
2 changed files with 116 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/helpers/eval-store.ts
+44
View File
@@ -37,6 +37,11 @@ export interface EvalTestEntry {
judge_scores?: Record<string, number>;
judge_reasoning?: string;
// Machine-readable diagnostics
exit_reason?: string; // 'success' | 'timeout' | 'error_max_turns' | 'exit_code_N'
timeout_at_turn?: number; // which turn was active when timeout hit
last_tool_call?: string; // e.g. "Write(review-output.md)"
// Outcome eval
detection_rate?: number;
false_positives?: number;
@@ -61,6 +66,7 @@ export interface EvalResult {
total_cost_usd: number;
total_duration_ms: number;
tests: EvalTestEntry[];
_partial?: boolean; // true for incremental saves, absent in final
}
export interface TestDelta {
@@ -374,6 +380,41 @@ export class EvalCollector {
addTest(entry: EvalTestEntry): void {
this.tests.push(entry);
this.savePartial();
}
/** Write incremental results after each test. Atomic write, non-fatal. */
savePartial(): void {
try {
const git = getGitInfo();
const version = getVersion();
const totalCost = this.tests.reduce((s, t) => s + t.cost_usd, 0);
const totalDuration = this.tests.reduce((s, t) => s + t.duration_ms, 0);
const passed = this.tests.filter(t => t.passed).length;
const partial: EvalResult = {
schema_version: SCHEMA_VERSION,
version,
branch: git.branch,
git_sha: git.sha,
timestamp: new Date().toISOString(),
hostname: os.hostname(),
tier: this.tier,
total_tests: this.tests.length,
passed,
failed: this.tests.length - passed,
total_cost_usd: Math.round(totalCost * 100) / 100,
total_duration_ms: totalDuration,
tests: this.tests,
_partial: true,
};
fs.mkdirSync(this.evalDir, { recursive: true });
const partialPath = path.join(this.evalDir, '_partial-e2e.json');
const tmp = partialPath + '.tmp';
fs.writeFileSync(tmp, JSON.stringify(partial, null, 2) + '\n');
fs.renameSync(tmp, partialPath);
} catch { /* non-fatal — partial saves are best-effort */ }
}
async finalize(): Promise<string> {
@@ -403,6 +444,9 @@ export class EvalCollector {
tests: this.tests,
};
// Delete partial file now that we're writing the final
try { fs.unlinkSync(path.join(this.evalDir, '_partial-e2e.json')); } catch { /* may not exist */ }
// Write eval file
fs.mkdirSync(this.evalDir, { recursive: true });
const dateStr = timestamp.replace(/[:.]/g, '').replace('T', '-').slice(0, 15);