feat(evals): env-driven lazy eval dir + shard-aware store and tooling

Importing eval-store no longer spawns the gstack-slug subprocess: the
module-level DEFAULT_EVAL_DIR constant is now a memoized defaultEvalDir()
resolved at collector construction. Resolution order: explicit
constructor arg, then GSTACK_EVAL_DIR, then slug detection — so the
sharded paid runner can point each shard child at its own
<evalDir>/shards/<slug>/ dir with plain env, no --preload.

Runs collected under a shards/ subdir record their slug in the eval
JSON (EvalResult.shard). findPreviousRun scans one shards/<slug>/ level
and prefers same-slug priors, so each shard baselines against its own
history instead of whichever shard flushed last. eval:list,
eval:summary, and eval:compare enumerate the same one level of shard
subdirs; eval:compare's no-arg mode also stops picking an in-progress
accumulator as the after-run.

eval-watch stays flat (documented follow-up): it tails a single dir for
live progress and gains nothing from per-shard baselines until the
runner emits a merged stream.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
(cherry picked from commit e1f53f7d9c7fe6b65877d843f2e25bd2e2d12ffd)

scripts/eval-compare.ts

@@ -10,12 +10,13 @@
 
 import * as fs from 'fs';
 import * as path from 'path';
-import * as os from 'os';
 import {
   findPreviousRun,
   compareEvalResults,
   formatComparison,
   getProjectEvalDir,
+  isPartialEval,
+  listEvalJsonFiles,
 } from '../test/helpers/eval-store';
 import type { EvalResult } from '../test/helpers/eval-store';
 
@@ -52,25 +53,34 @@ if (args.length === 2) {
   }
   beforeFile = prev;
 } else {
-  // No args — find two most recent of the same tier
-  let files: string[];
-  try {
-    files = fs.readdirSync(EVAL_DIR)
-      .filter(f => f.endsWith('.json'))
-      .sort()
-      .reverse();
-  } catch {
+  // No args — find two most recent of the same tier. Scans the flat dir plus
+  // one level of shards/<slug>/; in-progress accumulators are never the
+  // "after" run (comparing against a half-finished run says nothing).
+  const files = listEvalJsonFiles(EVAL_DIR)
+    .sort((a, b) => path.basename(b).localeCompare(path.basename(a)));
+
+  if (files.length === 0) {
     console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
     process.exit(0);
   }
-
   if (files.length < 2) {
     console.log('Need at least 2 eval runs to compare. Run evals again.');
     process.exit(0);
   }
 
-  // Most recent file
-  afterFile = path.join(EVAL_DIR, files[0]);
+  // Most recent finalized file
+  const latest = files.find(f => {
+    try {
+      return !isPartialEval(JSON.parse(fs.readFileSync(f, 'utf-8')), f);
+    } catch {
+      return false;
+    }
+  });
+  if (!latest) {
+    console.log('No completed eval runs yet. Run: EVALS=1 bun run test:evals');
+    process.exit(0);
+  }
+  afterFile = latest;
   const afterResult = loadResult(afterFile);
   const prev = findPreviousRun(EVAL_DIR, afterResult.tier, afterResult.branch, afterFile);
   if (!prev) {

scripts/eval-list.ts

@@ -6,9 +6,7 @@
  */
 
 import * as fs from 'fs';
-import * as path from 'path';
-import * as os from 'os';
-import { getProjectEvalDir } from '../test/helpers/eval-store';
+import { getProjectEvalDir, listEvalJsonFiles } from '../test/helpers/eval-store';
 
 const EVAL_DIR = getProjectEvalDir();
 
@@ -37,14 +35,8 @@ for (let i = 0; i < args.length; i++) {
   else if (args[i] === '--limit') { limit = parseLimit(args[++i]); }
 }
 
-// Read eval files
-let files: string[];
-try {
-  files = fs.readdirSync(EVAL_DIR).filter(f => f.endsWith('.json'));
-} catch {
-  console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
-  process.exit(0);
-}
+// Read eval files (flat dir plus one level of shards/<slug>/)
+const files = listEvalJsonFiles(EVAL_DIR);
 
 if (files.length === 0) {
   console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
@@ -68,7 +60,7 @@ interface RunSummary {
 const runs: RunSummary[] = [];
 for (const file of files) {
   try {
-    const data = JSON.parse(fs.readFileSync(path.join(EVAL_DIR, file), 'utf-8'));
+    const data = JSON.parse(fs.readFileSync(file, 'utf-8'));
     if (filterBranch && data.branch !== filterBranch) continue;
     if (filterTier && data.tier !== filterTier) continue;
     const totalTurns = (data.tests || []).reduce((s: number, t: any) => s + (t.turns_used || 0), 0);

scripts/eval-summary.ts

@@ -6,20 +6,13 @@
  */
 
 import * as fs from 'fs';
-import * as path from 'path';
-import * as os from 'os';
 import type { EvalResult } from '../test/helpers/eval-store';
-import { getProjectEvalDir } from '../test/helpers/eval-store';
+import { getProjectEvalDir, listEvalJsonFiles } from '../test/helpers/eval-store';
 
 const EVAL_DIR = getProjectEvalDir();
 
-let files: string[];
-try {
-  files = fs.readdirSync(EVAL_DIR).filter(f => f.endsWith('.json'));
-} catch {
-  console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
-  process.exit(0);
-}
+// Flat dir plus one level of shards/<slug>/
+const files = listEvalJsonFiles(EVAL_DIR);
 
 if (files.length === 0) {
   console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
@@ -30,7 +23,7 @@ if (files.length === 0) {
 const results: EvalResult[] = [];
 for (const file of files) {
   try {
-    results.push(JSON.parse(fs.readFileSync(path.join(EVAL_DIR, file), 'utf-8')));
+    results.push(JSON.parse(fs.readFileSync(file, 'utf-8')));
   } catch { continue; }
 }