feat: eval-watch dashboard + observability unit tests (15 tests, 11 codepaths)

eval-watch: live terminal dashboard reads heartbeat + partial file every 1s, shows completed/running tests, stale detection (>10min), --tail flag for progress.log tail. Pure renderDashboard() function for testability. observability.test.ts: unit tests for sanitizeTestName, heartbeat schema, progress.log format, NDJSON file naming, savePartial() with _partial flag, finalize() cleanup, diagnostic fields, watcher rendering, stale detection, and non-fatal I/O guarantees. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-05-01 19:25:10 +02:00 · 2026-03-14 11:04:40 -05:00
parent 510a8d8dda
commit 029a7c2a37
2 changed files with 435 additions and 0 deletions
@@ -0,0 +1,153 @@
+/**
+ * Live E2E test watcher dashboard.
+ *
+ * Reads heartbeat (e2e-live.json) for current test status and
+ * partial eval results (_partial-e2e.json) for completed tests.
+ * Renders a terminal dashboard every 1s.
+ *
+ * Usage: bun run eval:watch [--tail]
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+
+const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev');
+const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json');
+const PARTIAL_PATH = path.join(GSTACK_DEV_DIR, 'evals', '_partial-e2e.json');
+const STALE_THRESHOLD_SEC = 600; // 10 minutes
+
+export interface HeartbeatData {
+  runId: string;
+  startedAt: string;
+  currentTest: string;
+  status: string;
+  turn: number;
+  toolCount: number;
+  lastTool: string;
+  lastToolAt: string;
+  elapsedSec: number;
+}
+
+export interface PartialData {
+  tests: Array<{
+    name: string;
+    passed: boolean;
+    cost_usd: number;
+    duration_ms: number;
+    turns_used?: number;
+    exit_reason?: string;
+  }>;
+  total_cost_usd: number;
+  _partial?: boolean;
+}
+
+/** Read and parse a JSON file, returning null on any error. */
+function readJSON<T>(filePath: string): T | null {
+  try {
+    return JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+  } catch {
+    return null;
+  }
+}
+
+/** Format seconds as Xm Ys */
+function formatDuration(sec: number): string {
+  if (sec < 60) return `${sec}s`;
+  const m = Math.floor(sec / 60);
+  const s = sec % 60;
+  return `${m}m ${s}s`;
+}
+
+/** Render dashboard from heartbeat + partial data. Pure function for testability. */
+export function renderDashboard(heartbeat: HeartbeatData | null, partial: PartialData | null): string {
+  const lines: string[] = [];
+
+  if (!heartbeat && !partial) {
+    lines.push('E2E Watch — No active run detected');
+    lines.push('');
+    lines.push(`Heartbeat: ${HEARTBEAT_PATH} (not found)`);
+    lines.push(`Partial:   ${PARTIAL_PATH} (not found)`);
+    lines.push('');
+    lines.push('Start a run with: EVALS=1 bun test test/skill-e2e.test.ts');
+    return lines.join('\n');
+  }
+
+  const runId = heartbeat?.runId || 'unknown';
+  const elapsed = heartbeat?.elapsedSec || 0;
+  lines.push(`E2E Watch \u2014 Run ${runId} \u2014 ${formatDuration(elapsed)}`);
+  lines.push('\u2550'.repeat(55));
+
+  // Completed tests from partial
+  if (partial?.tests) {
+    for (const t of partial.tests) {
+      const icon = t.passed ? '\u2713' : '\u2717';
+      const cost = `$${t.cost_usd.toFixed(2)}`;
+      const dur = `${Math.round(t.duration_ms / 1000)}s`;
+      const turns = t.turns_used !== undefined ? `${t.turns_used} turns` : '';
+      const name = t.name.length > 30 ? t.name.slice(0, 27) + '...' : t.name.padEnd(30);
+      lines.push(` ${icon} ${name}  ${cost.padStart(6)}  ${dur.padStart(5)}  ${turns}`);
+    }
+  }
+
+  // Current test from heartbeat
+  if (heartbeat && heartbeat.status === 'running') {
+    const name = heartbeat.currentTest.length > 30
+      ? heartbeat.currentTest.slice(0, 27) + '...'
+      : heartbeat.currentTest.padEnd(30);
+    lines.push(` \u29D6 ${name}  ${formatDuration(heartbeat.elapsedSec).padStart(6)}  turn ${heartbeat.turn}   last: ${heartbeat.lastTool}`);
+
+    // Stale detection
+    const lastToolTime = new Date(heartbeat.lastToolAt).getTime();
+    const staleSec = Math.round((Date.now() - lastToolTime) / 1000);
+    if (staleSec > STALE_THRESHOLD_SEC) {
+      lines.push(` \u26A0  STALE: last tool call was ${formatDuration(staleSec)} ago \u2014 run may have crashed`);
+    }
+  }
+
+  lines.push('\u2500'.repeat(55));
+
+  // Summary
+  const completedCount = partial?.tests?.length || 0;
+  const totalCost = partial?.total_cost_usd || 0;
+  const running = heartbeat?.status === 'running' ? 1 : 0;
+  lines.push(` Completed: ${completedCount}  Running: ${running}  Cost: $${totalCost.toFixed(2)}  Elapsed: ${formatDuration(elapsed)}`);
+
+  if (heartbeat?.runId) {
+    const logPath = path.join(GSTACK_DEV_DIR, 'e2e-runs', heartbeat.runId, 'progress.log');
+    lines.push(` Logs: ${logPath}`);
+  }
+
+  return lines.join('\n');
+}
+
+// --- Main ---
+
+if (import.meta.main) {
+  const showTail = process.argv.includes('--tail');
+
+  const render = () => {
+    const heartbeat = readJSON<HeartbeatData>(HEARTBEAT_PATH);
+    const partial = readJSON<PartialData>(PARTIAL_PATH);
+
+    // Clear screen
+    process.stdout.write('\x1B[2J\x1B[H');
+    process.stdout.write(renderDashboard(heartbeat, partial) + '\n');
+
+    // --tail: show last 10 lines of progress.log
+    if (showTail && heartbeat?.runId) {
+      const logPath = path.join(GSTACK_DEV_DIR, 'e2e-runs', heartbeat.runId, 'progress.log');
+      try {
+        const content = fs.readFileSync(logPath, 'utf-8');
+        const tail = content.split('\n').filter(l => l.trim()).slice(-10);
+        process.stdout.write('\nRecent progress:\n');
+        for (const line of tail) {
+          process.stdout.write(line + '\n');
+        }
+      } catch { /* log file may not exist yet */ }
+    }
+  };
+
+  render();
+  setInterval(render, 1000);
+}