feat(evals): env-driven lazy eval dir + shard-aware store and tooling

Importing eval-store no longer spawns the gstack-slug subprocess: the
module-level DEFAULT_EVAL_DIR constant is now a memoized defaultEvalDir()
resolved at collector construction. Resolution order: explicit
constructor arg, then GSTACK_EVAL_DIR, then slug detection — so the
sharded paid runner can point each shard child at its own
<evalDir>/shards/<slug>/ dir with plain env, no --preload.

Runs collected under a shards/ subdir record their slug in the eval
JSON (EvalResult.shard). findPreviousRun scans one shards/<slug>/ level
and prefers same-slug priors, so each shard baselines against its own
history instead of whichever shard flushed last. eval:list,
eval:summary, and eval:compare enumerate the same one level of shard
subdirs; eval:compare's no-arg mode also stops picking an in-progress
accumulator as the after-run.

eval-watch stays flat (documented follow-up): it tails a single dir for
live progress and gains nothing from per-shard baselines until the
runner emits a merged stream.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
(cherry picked from commit e1f53f7d9c7fe6b65877d843f2e25bd2e2d12ffd)

scripts/eval-compare.ts

@@ -10,12 +10,13 @@
 
 import * as fs from 'fs';
 import * as path from 'path';
-import * as os from 'os';
 import {
   findPreviousRun,
   compareEvalResults,
   formatComparison,
   getProjectEvalDir,
+  isPartialEval,
+  listEvalJsonFiles,
 } from '../test/helpers/eval-store';
 import type { EvalResult } from '../test/helpers/eval-store';
 
@@ -52,25 +53,34 @@ if (args.length === 2) {
   }
   beforeFile = prev;
 } else {
-  // No args — find two most recent of the same tier
+  // No args — find two most recent of the same tier. Scans the flat dir plus
-  let files: string[];
+  // one level of shards/<slug>/; in-progress accumulators are never the
-  try {
+  // "after" run (comparing against a half-finished run says nothing).
-    files = fs.readdirSync(EVAL_DIR)
+  const files = listEvalJsonFiles(EVAL_DIR)
-      .filter(f => f.endsWith('.json'))
+    .sort((a, b) => path.basename(b).localeCompare(path.basename(a)));
-      .sort()
+
-      .reverse();
+  if (files.length === 0) {
-  } catch {
     console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
     process.exit(0);
   }
-
   if (files.length < 2) {
     console.log('Need at least 2 eval runs to compare. Run evals again.');
     process.exit(0);
   }
 
-  // Most recent file
+  // Most recent finalized file
-  afterFile = path.join(EVAL_DIR, files[0]);
+  const latest = files.find(f => {
+    try {
+      return !isPartialEval(JSON.parse(fs.readFileSync(f, 'utf-8')), f);
+    } catch {
+      return false;
+    }
+  });
+  if (!latest) {
+    console.log('No completed eval runs yet. Run: EVALS=1 bun run test:evals');
+    process.exit(0);
+  }
+  afterFile = latest;
   const afterResult = loadResult(afterFile);
   const prev = findPreviousRun(EVAL_DIR, afterResult.tier, afterResult.branch, afterFile);
   if (!prev) {

scripts/eval-list.ts

@@ -6,9 +6,7 @@
  */
 
 import * as fs from 'fs';
-import * as path from 'path';
+import { getProjectEvalDir, listEvalJsonFiles } from '../test/helpers/eval-store';
-import * as os from 'os';
-import { getProjectEvalDir } from '../test/helpers/eval-store';
 
 const EVAL_DIR = getProjectEvalDir();
 
@@ -37,14 +35,8 @@ for (let i = 0; i < args.length; i++) {
   else if (args[i] === '--limit') { limit = parseLimit(args[++i]); }
 }
 
-// Read eval files
+// Read eval files (flat dir plus one level of shards/<slug>/)
-let files: string[];
+const files = listEvalJsonFiles(EVAL_DIR);
-try {
-  files = fs.readdirSync(EVAL_DIR).filter(f => f.endsWith('.json'));
-} catch {
-  console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
-  process.exit(0);
-}
 
 if (files.length === 0) {
   console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
@@ -68,7 +60,7 @@ interface RunSummary {
 const runs: RunSummary[] = [];
 for (const file of files) {
   try {
-    const data = JSON.parse(fs.readFileSync(path.join(EVAL_DIR, file), 'utf-8'));
+    const data = JSON.parse(fs.readFileSync(file, 'utf-8'));
     if (filterBranch && data.branch !== filterBranch) continue;
     if (filterTier && data.tier !== filterTier) continue;
     const totalTurns = (data.tests || []).reduce((s: number, t: any) => s + (t.turns_used || 0), 0);

scripts/eval-summary.ts

@@ -6,20 +6,13 @@
  */
 
 import * as fs from 'fs';
-import * as path from 'path';
-import * as os from 'os';
 import type { EvalResult } from '../test/helpers/eval-store';
-import { getProjectEvalDir } from '../test/helpers/eval-store';
+import { getProjectEvalDir, listEvalJsonFiles } from '../test/helpers/eval-store';
 
 const EVAL_DIR = getProjectEvalDir();
 
-let files: string[];
+// Flat dir plus one level of shards/<slug>/
-try {
+const files = listEvalJsonFiles(EVAL_DIR);
-  files = fs.readdirSync(EVAL_DIR).filter(f => f.endsWith('.json'));
-} catch {
-  console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
-  process.exit(0);
-}
 
 if (files.length === 0) {
   console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
@@ -30,7 +23,7 @@ if (files.length === 0) {
 const results: EvalResult[] = [];
 for (const file of files) {
   try {
-    results.push(JSON.parse(fs.readFileSync(path.join(EVAL_DIR, file), 'utf-8')));
+    results.push(JSON.parse(fs.readFileSync(file, 'utf-8')));
   } catch { continue; }
 }
 

test/helpers/eval-store.test.ts

@@ -182,6 +182,59 @@ describe('EvalCollector', () => {
   });
 });
 
+// --- GSTACK_EVAL_DIR + shard slug tests ---
+
+describe('EvalCollector eval-dir resolution', () => {
+  const savedEnv = process.env.GSTACK_EVAL_DIR;
+
+  afterEach(() => {
+    if (savedEnv === undefined) delete process.env.GSTACK_EVAL_DIR;
+    else process.env.GSTACK_EVAL_DIR = savedEnv;
+  });
+
+  test('honors GSTACK_EVAL_DIR set after import — no --preload needed', async () => {
+    // The default eval dir must resolve lazily at construction, not at module
+    // load: the sharded runner sets GSTACK_EVAL_DIR in each shard child's env
+    // and shard tests import this module long before any collector exists.
+    const envDir = path.join(tmpDir, 'env-dir');
+    process.env.GSTACK_EVAL_DIR = envDir;
+    const collector = new EvalCollector('e2e');
+    collector.addTest(makeEntry());
+    await captureStderr(async () => { await collector.finalize(); });
+    expect(fs.readdirSync(envDir).filter(f => !f.startsWith('_partial'))).toHaveLength(1);
+  });
+
+  test('explicit constructor arg beats GSTACK_EVAL_DIR', async () => {
+    process.env.GSTACK_EVAL_DIR = path.join(tmpDir, 'env-dir');
+    const explicit = path.join(tmpDir, 'explicit');
+    const collector = new EvalCollector('e2e', explicit);
+    collector.addTest(makeEntry());
+    await captureStderr(async () => { await collector.finalize(); });
+    expect(fs.existsSync(path.join(tmpDir, 'env-dir'))).toBe(false);
+    expect(fs.readdirSync(explicit).length).toBeGreaterThan(0);
+  });
+
+  test('writes the shard slug when the eval dir is a shards/ subdir', async () => {
+    const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa');
+    const collector = new EvalCollector('e2e', shardDir);
+    collector.addTest(makeEntry());
+    await captureStderr(async () => { await collector.finalize(); });
+
+    const partial = JSON.parse(fs.readFileSync(path.join(shardDir, '_partial-e2e.json'), 'utf-8'));
+    expect(partial.shard).toBe('skill-e2e-qa');
+    const final = fs.readdirSync(shardDir).find(f => !f.startsWith('_partial'))!;
+    expect(JSON.parse(fs.readFileSync(path.join(shardDir, final), 'utf-8')).shard).toBe('skill-e2e-qa');
+  });
+
+  test('writes no shard slug for a flat eval dir', async () => {
+    const collector = new EvalCollector('e2e', tmpDir);
+    collector.addTest(makeEntry());
+    await captureStderr(async () => { await collector.finalize(); });
+    const final = fs.readdirSync(tmpDir).find(f => f.endsWith('.json') && !f.startsWith('_partial'))!;
+    expect(JSON.parse(fs.readFileSync(path.join(tmpDir, final), 'utf-8')).shard).toBeUndefined();
+  });
+});
+
 // --- judgePassed tests ---
 
 describe('judgePassed', () => {
@@ -347,6 +400,57 @@ describe('findPreviousRun', () => {
     const result = findPreviousRun(tmpDir, 'e2e', 'main', 'current.json');
     expect(result).toBeNull(); // only llm-judge file, looking for e2e
   });
+
+  test('a shard run prefers its own shard history over newer other-shard or flat priors', () => {
+    const mine = path.join(tmpDir, 'shards', 'skill-e2e-qa');
+    const other = path.join(tmpDir, 'shards', 'codex-e2e');
+    fs.mkdirSync(mine, { recursive: true });
+    fs.mkdirSync(other, { recursive: true });
+    // Same-shard prior — oldest of the three, must still win.
+    fs.writeFileSync(
+      path.join(mine, '0.3.4-main-e2e-20260311-100000.json'),
+      JSON.stringify(makeResult({ timestamp: '2026-03-11T10:00:00Z' })),
+    );
+    fs.writeFileSync(
+      path.join(other, '0.3.5-main-e2e-20260312-100000.json'),
+      JSON.stringify(makeResult({ timestamp: '2026-03-12T10:00:00Z' })),
+    );
+    fs.writeFileSync(
+      path.join(tmpDir, '0.3.6-main-e2e-20260313-100000.json'),
+      JSON.stringify(makeResult({ timestamp: '2026-03-13T10:00:00Z' })),
+    );
+
+    const result = findPreviousRun(tmpDir, 'e2e', 'main', path.join(mine, 'current.json'));
+    expect(result).toContain(path.join('shards', 'skill-e2e-qa'));
+  });
+
+  test('a flat run prefers flat history over a newer shard prior', () => {
+    const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa');
+    fs.mkdirSync(shardDir, { recursive: true });
+    fs.writeFileSync(
+      path.join(shardDir, '0.3.6-main-e2e-20260314-100000.json'),
+      JSON.stringify(makeResult({ timestamp: '2026-03-14T10:00:00Z' })),
+    );
+    fs.writeFileSync(
+      path.join(tmpDir, '0.3.5-main-e2e-20260312-100000.json'),
+      JSON.stringify(makeResult({ timestamp: '2026-03-12T10:00:00Z' })),
+    );
+
+    const result = findPreviousRun(tmpDir, 'e2e', 'main', path.join(tmpDir, 'current.json'));
+    expect(result).toContain('0.3.5-main-e2e');
+  });
+
+  test('falls back to a shard prior when the flat dir has no candidate', () => {
+    const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa');
+    fs.mkdirSync(shardDir, { recursive: true });
+    fs.writeFileSync(
+      path.join(shardDir, '0.3.6-main-e2e-20260314-100000.json'),
+      JSON.stringify(makeResult({ timestamp: '2026-03-14T10:00:00Z' })),
+    );
+
+    const result = findPreviousRun(tmpDir, 'e2e', 'main', path.join(tmpDir, 'current.json'));
+    expect(result).toContain(path.join('shards', 'skill-e2e-qa'));
+  });
 });
 
 // --- isPartialEval tests ---

test/helpers/eval-store.ts

@@ -39,7 +39,16 @@ export function getProjectEvalDir(): string {
   return LEGACY_EVAL_DIR;
 }
 
-const DEFAULT_EVAL_DIR = getProjectEvalDir();
+/**
+ * Lazy + memoized so importing this module never spawns the gstack-slug
+ * subprocess. Callers that pass an explicit dir or set GSTACK_EVAL_DIR
+ * (the sharded paid runner does, per shard) never pay for slug detection.
+ */
+let memoizedDefaultEvalDir: string | null = null;
+function defaultEvalDir(): string {
+  if (memoizedDefaultEvalDir === null) memoizedDefaultEvalDir = getProjectEvalDir();
+  return memoizedDefaultEvalDir;
+}
 
 // --- Interfaces ---
 
@@ -104,6 +113,8 @@ export interface EvalResult {
   total_duration_ms: number;
   wall_clock_ms?: number;     // wall-clock from collector creation to finalization (shows parallelism)
   tests: EvalTestEntry[];
+  /** Shard slug when the run was collected under <evalDir>/shards/<slug>/. */
+  shard?: string;
   _partial?: boolean;  // true for incremental saves, absent in final
 }
 
@@ -184,6 +195,16 @@ export function listEvalJsonFiles(evalDir: string): string[] {
   return files;
 }
 
+/**
+ * Shard slug for an eval dir: when the dir is directly under a `shards/`
+ * directory (the sharded paid runner's per-shard GSTACK_EVAL_DIR layout),
+ * the dir name is the slug; otherwise null.
+ */
+export function shardSlugOfEvalDir(evalDir: string): string | null {
+  const normalized = path.resolve(evalDir);
+  return path.basename(path.dirname(normalized)) === 'shards' ? path.basename(normalized) : null;
+}
+
 /**
  * Find the most recent finalized (non-partial) eval file for a tier, scanning
  * `evalDir` and one level of `shards/<slug>/` subdirs. Shared by the budget
@@ -246,7 +267,9 @@ export function extractToolSummary(transcript: any[]): Record<string, number> {
 
 /**
  * Find the most recent prior COMPLETED eval file for comparison.
- * Prefers same branch, falls back to any branch.
+ * Scans the eval dir plus one level of `shards/<slug>/` subdirs. Prefers
+ * same shard slug (a shard's own history over another shard's or the flat
+ * dir's), then same branch, then falls back to anything.
  *
  * In-progress accumulators (`_partial: true`, written by savePartial after every
  * test) are never candidates: the current run's own partial carries the current
@@ -260,25 +283,22 @@ export function findPreviousRun(
   branch: string,
   excludeFile: string,
 ): string | null {
-  let files: string[];
-  try {
-    files = fs.readdirSync(evalDir).filter(f => f.endsWith('.json'));
-  } catch {
-    return null; // dir doesn't exist
-  }
-
   // Parse top-level fields from each file (cheap — no full tests array needed)
-  const entries: Array<{ file: string; branch: string; timestamp: string }> = [];
+  const entries: Array<{ file: string; branch: string; timestamp: string; shard: string | null }> = [];
-  for (const file of files) {
+  for (const fullPath of listEvalJsonFiles(evalDir)) {
-    if (file === path.basename(excludeFile)) continue;
+    if (path.resolve(fullPath) === path.resolve(excludeFile)) continue;
-    const fullPath = path.join(evalDir, file);
     try {
       const raw = fs.readFileSync(fullPath, 'utf-8');
       // Quick parse — only grab the fields we need
       const data = JSON.parse(raw);
-      if (isPartialEval(data, file)) continue; // in-progress run, not a baseline
+      if (isPartialEval(data, fullPath)) continue; // in-progress run, not a baseline
       if (data.tier !== tier) continue;
-      entries.push({ file: fullPath, branch: data.branch || '', timestamp: data.timestamp || '' });
+      entries.push({
+        file: fullPath,
+        branch: data.branch || '',
+        timestamp: data.timestamp || '',
+        shard: data.shard || shardSlugOfEvalDir(path.dirname(fullPath)),
+      });
     } catch { continue; }
   }
 
@@ -287,11 +307,17 @@ export function findPreviousRun(
   // Sort by timestamp descending
   entries.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
 
-  // Prefer same branch
+  // Prefer same shard slug (null = the flat dir), then same branch, then any.
-  const sameBranch = entries.find(e => e.branch === branch);
+  const targetShard = shardSlugOfEvalDir(path.dirname(excludeFile));
-  if (sameBranch) return sameBranch.file;
+  const preferences: Array<(e: typeof entries[number]) => boolean> = [
-
+    e => e.shard === targetShard && e.branch === branch,
-  // Fallback: any branch
+    e => e.shard === targetShard,
+    e => e.branch === branch,
+  ];
+  for (const matches of preferences) {
+    const hit = entries.find(matches);
+    if (hit) return hit.file;
+  }
   return entries[0].file;
 }
 
@@ -743,11 +769,13 @@ export class EvalCollector {
   private tests: EvalTestEntry[] = [];
   private finalized = false;
   private evalDir: string;
+  private shard: string | null;
   private createdAt = Date.now();
 
   constructor(tier: 'e2e' | 'llm-judge', evalDir?: string) {
     this.tier = tier;
-    this.evalDir = evalDir || DEFAULT_EVAL_DIR;
+    this.evalDir = evalDir || process.env.GSTACK_EVAL_DIR || defaultEvalDir();
+    this.shard = shardSlugOfEvalDir(this.evalDir);
   }
 
   addTest(entry: EvalTestEntry): void {
@@ -778,6 +806,7 @@ export class EvalCollector {
         total_cost_usd: Math.round(totalCost * 100) / 100,
         total_duration_ms: totalDuration,
         tests: this.tests,
+        ...(this.shard ? { shard: this.shard } : {}),
         _partial: true,
       };
 
@@ -815,6 +844,7 @@ export class EvalCollector {
       total_duration_ms: totalDuration,
       wall_clock_ms: Date.now() - this.createdAt,
       tests: this.tests,
+      ...(this.shard ? { shard: this.shard } : {}),
     };
 
     // Write eval file