gstack/scripts/eval-watch.ts

/**
 * Live E2E test watcher dashboard.
 *
 * Reads heartbeat (e2e-live.json) for current test status and
 * partial eval results (_partial-e2e*.json) for completed tests.
 * Renders a terminal dashboard every 1s.
 *
 * Usage: bun run eval:watch [--tail]
 */

import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { evalEntryOutcome, getProjectEvalDir } from '../test/helpers/eval-store';
import type { EvalTestEntry } from '../test/helpers/eval-store';

const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev');
// Heartbeat + per-run progress logs are GLOBAL by design — session-runner.ts
// writes ~/.gstack-dev/e2e-live.json regardless of project ("heartbeat stays
// global"). The PARTIAL file is per-project: EvalCollector writes it into
// getProjectEvalDir() (or GSTACK_EVAL_DIR), so watching the legacy global
// path missed it whenever slug detection succeeded — i.e. the normal case.
const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json');
const EVAL_DIR = process.env.GSTACK_EVAL_DIR || getProjectEvalDir();
const PARTIAL_PATH = path.join(EVAL_DIR, '_partial-e2e*.json');
const STALE_THRESHOLD_SEC = 600; // 10 minutes

export interface HeartbeatData {
  runId: string;
  pid?: number;
  startedAt: string;
  currentTest: string;
  status: string;
  turn: number;
  toolCount: number;
  lastTool: string;
  lastToolAt: string;
  elapsedSec: number;
}

export interface PartialData {
  tests: Array<Partial<EvalTestEntry> & Pick<EvalTestEntry, 'name' | 'passed' | 'cost_usd' | 'duration_ms'>>;
  total_cost_usd: number;
  _partial?: boolean;
}

/** Read and parse a JSON file, returning null on any error. */
function readJSON<T>(filePath: string): T | null {
  try {
    return JSON.parse(fs.readFileSync(filePath, 'utf-8'));
  } catch {
    return null;
  }
}

/** Read the legacy collector and suite collectors in this shard, never finals. */
export function readPartialResults(evalDir = EVAL_DIR): PartialData | null {
  let names: string[];
  try { names = fs.readdirSync(evalDir).sort(); } catch { return null; }
  const partials = names.filter(name => /^_partial-e2e(?:-[a-z0-9]+(?:-[a-z0-9]+)*)?\.json$/.test(name))
    .map(name => ({ name, data: readJSON<PartialData>(path.join(evalDir, name)) }))
    .filter((item): item is { name: string; data: PartialData } => Array.isArray(item.data?.tests));
  if (partials.length === 0) return null;
  if (partials.length === 1) return partials[0].data;
  const named = partials.filter(item => item.name !== '_partial-e2e.json').flatMap(item => item.data.tests);
  const namedSuites = new Set(named.map(test => test.suite ?? null));
  const legacy = partials.find(item => item.name === '_partial-e2e.json')?.data.tests ?? [];
  // A legacy accumulator can remain after adopting suite files. Prefer the
  // entire current suite snapshot, including its own retries. Old retries and
  // retired tests must not leak back in from the legacy snapshot.
  const tests = [...legacy.filter(test => !namedSuites.has(test.suite ?? null)), ...named];
  return { tests, total_cost_usd: tests.reduce((sum, test) => sum + test.cost_usd, 0), _partial: true };
}

/** Check if a process is alive (signal 0 = existence check, doesn't kill). */
function isProcessAlive(pid: number): boolean {
  try {
    process.kill(pid, 0);
    return true;
  } catch {
    return false;
  }
}

/** Format seconds as Xm Ys */
function formatDuration(sec: number): string {
  if (sec < 60) return `${sec}s`;
  const m = Math.floor(sec / 60);
  const s = sec % 60;
  return `${m}m ${s}s`;
}

/** Render dashboard from heartbeat + partial data. Pure function for testability. */
export function renderDashboard(heartbeat: HeartbeatData | null, partial: PartialData | null): string {
  const lines: string[] = [];

  if (!heartbeat && !partial) {
    lines.push('E2E Watch — No active run detected');
    lines.push('');
    lines.push(`Heartbeat: ${HEARTBEAT_PATH} (not found)`);
    lines.push(`Partial:   ${PARTIAL_PATH} (not found)`);
    lines.push('');
    lines.push('Start a run with: EVALS=1 bun test test/skill-e2e-*.test.ts');
    return lines.join('\n');
  }

  const runId = heartbeat?.runId || 'unknown';
  const elapsed = heartbeat?.elapsedSec || 0;
  lines.push(`E2E Watch \u2014 Run ${runId} \u2014 ${formatDuration(elapsed)}`);
  lines.push('\u2550'.repeat(55));

  // Completed tests from partial
  if (partial?.tests) {
    for (const t of partial.tests) {
      const manual = evalEntryOutcome(t) === 'manual-review';
      const icon = manual ? 'M' : evalEntryOutcome(t) === 'passed' ? '\u2713' : '\u2717';
      const cost = `$${t.cost_usd.toFixed(2)}`;
      const dur = `${Math.round(t.duration_ms / 1000)}s`;
      const turns = t.turns_used !== undefined ? `${t.turns_used} turns` : '';
      const name = t.name.length > 30 ? t.name.slice(0, 27) + '...' : t.name.padEnd(30);
      const approval = manual ? ` MANUAL/unscored; approved by ${t.manual_review!.approval.approved_by} (${t.manual_review!.approval.approval_url})` : '';
      lines.push(` ${icon} ${name}  ${cost.padStart(6)}  ${dur.padStart(5)}  ${turns}${approval}`);
    }
  }

  // Current test from heartbeat
  if (heartbeat && heartbeat.status === 'running') {
    const name = heartbeat.currentTest.length > 30
      ? heartbeat.currentTest.slice(0, 27) + '...'
      : heartbeat.currentTest.padEnd(30);
    lines.push(` \u29D6 ${name}  ${formatDuration(heartbeat.elapsedSec).padStart(6)}  turn ${heartbeat.turn}   last: ${heartbeat.lastTool}`);

    // Stale detection
    const lastToolTime = new Date(heartbeat.lastToolAt).getTime();
    const staleSec = Math.round((Date.now() - lastToolTime) / 1000);
    if (staleSec > STALE_THRESHOLD_SEC) {
      lines.push(` \u26A0  STALE: last tool call was ${formatDuration(staleSec)} ago \u2014 run may have crashed`);
    }
  }

  lines.push('\u2500'.repeat(55));

  // Summary
  const completedCount = partial?.tests?.length || 0;
  const totalCost = partial?.total_cost_usd || 0;
  const running = heartbeat?.status === 'running' ? 1 : 0;
  lines.push(` Completed: ${completedCount}  Running: ${running}  Cost: $${totalCost.toFixed(2)}  Elapsed: ${formatDuration(elapsed)}`);
  const manualAccepted = partial?.tests?.filter(t => evalEntryOutcome(t) === 'manual-review').length ?? 0;
  if (manualAccepted) lines.push(` Manual accepted: ${manualAccepted} unscored provider refusal(s)`);

  if (heartbeat?.runId) {
    const logPath = path.join(GSTACK_DEV_DIR, 'e2e-runs', heartbeat.runId, 'progress.log');
    lines.push(` Logs: ${logPath}`);
  }

  return lines.join('\n');
}

// --- Main ---

if (import.meta.main) {
  const showTail = process.argv.includes('--tail');

  const render = () => {
    let heartbeat = readJSON<HeartbeatData>(HEARTBEAT_PATH);
    const partial = readPartialResults();

    // Auto-clear heartbeat if the process is dead
    if (heartbeat?.pid && !isProcessAlive(heartbeat.pid)) {
      try { fs.unlinkSync(HEARTBEAT_PATH); } catch { /* already gone */ }
      process.stdout.write('\x1B[2J\x1B[H');
      process.stdout.write(`Cleared stale heartbeat — PID ${heartbeat.pid} is no longer running.\n\n`);
      heartbeat = null;
    }

    // Clear screen
    process.stdout.write('\x1B[2J\x1B[H');
    process.stdout.write(renderDashboard(heartbeat, partial) + '\n');

    // --tail: show last 10 lines of progress.log
    if (showTail && heartbeat?.runId) {
      const logPath = path.join(GSTACK_DEV_DIR, 'e2e-runs', heartbeat.runId, 'progress.log');
      try {
        const content = fs.readFileSync(logPath, 'utf-8');
        const tail = content.split('\n').filter(l => l.trim()).slice(-10);
        process.stdout.write('\nRecent progress:\n');
        for (const line of tail) {
          process.stdout.write(line + '\n');
        }
      } catch { /* log file may not exist yet */ }
    }
  };

  render();
  setInterval(render, 1000);
}