mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-09 22:48:57 +02:00
Garry Tan
9ff2b9b162
Importing eval-store no longer spawns the gstack-slug subprocess: the module-level DEFAULT_EVAL_DIR constant is now a memoized defaultEvalDir() resolved at collector construction. Resolution order: explicit constructor arg, then GSTACK_EVAL_DIR, then slug detection — so the sharded paid runner can point each shard child at its own <evalDir>/shards/<slug>/ dir with plain env, no --preload. Runs collected under a shards/ subdir record their slug in the eval JSON (EvalResult.shard). findPreviousRun scans one shards/<slug>/ level and prefers same-slug priors, so each shard baselines against its own history instead of whichever shard flushed last. eval:list, eval:summary, and eval:compare enumerate the same one level of shard subdirs; eval:compare's no-arg mode also stops picking an in-progress accumulator as the after-run. eval-watch stays flat (documented follow-up): it tails a single dir for live progress and gains nothing from per-shard baselines until the runner emits a merged stream. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> (cherry picked from commit e1f53f7d9c7fe6b65877d843f2e25bd2e2d12ffd)
108 lines
3.4 KiB
TypeScript
108 lines
3.4 KiB
TypeScript
#!/usr/bin/env bun
|
|
/**
|
|
* Compare two eval runs from ~/.gstack-dev/evals/
|
|
*
|
|
* Usage:
|
|
* bun run eval:compare # compare two most recent of same tier
|
|
* bun run eval:compare <file> # compare file against its predecessor
|
|
* bun run eval:compare <file-a> <file-b> # compare two specific files
|
|
*/
|
|
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
import {
|
|
findPreviousRun,
|
|
compareEvalResults,
|
|
formatComparison,
|
|
getProjectEvalDir,
|
|
isPartialEval,
|
|
listEvalJsonFiles,
|
|
} from '../test/helpers/eval-store';
|
|
import type { EvalResult } from '../test/helpers/eval-store';
|
|
|
|
const EVAL_DIR = getProjectEvalDir();
|
|
|
|
function loadResult(filepath: string): EvalResult {
|
|
// Resolve relative to EVAL_DIR if not absolute
|
|
const resolved = path.isAbsolute(filepath) ? filepath : path.join(EVAL_DIR, filepath);
|
|
if (!fs.existsSync(resolved)) {
|
|
console.error(`File not found: ${resolved}`);
|
|
process.exit(1);
|
|
}
|
|
return JSON.parse(fs.readFileSync(resolved, 'utf-8'));
|
|
}
|
|
|
|
const args = process.argv.slice(2);
|
|
|
|
let beforeFile: string;
|
|
let afterFile: string;
|
|
|
|
if (args.length === 2) {
|
|
// Two explicit files
|
|
beforeFile = args[0];
|
|
afterFile = args[1];
|
|
} else if (args.length === 1) {
|
|
// One file — find its predecessor
|
|
afterFile = args[0];
|
|
const resolved = path.isAbsolute(afterFile) ? afterFile : path.join(EVAL_DIR, afterFile);
|
|
const afterResult = loadResult(resolved);
|
|
const prev = findPreviousRun(EVAL_DIR, afterResult.tier, afterResult.branch, resolved);
|
|
if (!prev) {
|
|
console.log('No previous run found to compare against.');
|
|
process.exit(0);
|
|
}
|
|
beforeFile = prev;
|
|
} else {
|
|
// No args — find two most recent of the same tier. Scans the flat dir plus
|
|
// one level of shards/<slug>/; in-progress accumulators are never the
|
|
// "after" run (comparing against a half-finished run says nothing).
|
|
const files = listEvalJsonFiles(EVAL_DIR)
|
|
.sort((a, b) => path.basename(b).localeCompare(path.basename(a)));
|
|
|
|
if (files.length === 0) {
|
|
console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
|
|
process.exit(0);
|
|
}
|
|
if (files.length < 2) {
|
|
console.log('Need at least 2 eval runs to compare. Run evals again.');
|
|
process.exit(0);
|
|
}
|
|
|
|
// Most recent finalized file
|
|
const latest = files.find(f => {
|
|
try {
|
|
return !isPartialEval(JSON.parse(fs.readFileSync(f, 'utf-8')), f);
|
|
} catch {
|
|
return false;
|
|
}
|
|
});
|
|
if (!latest) {
|
|
console.log('No completed eval runs yet. Run: EVALS=1 bun run test:evals');
|
|
process.exit(0);
|
|
}
|
|
afterFile = latest;
|
|
const afterResult = loadResult(afterFile);
|
|
const prev = findPreviousRun(EVAL_DIR, afterResult.tier, afterResult.branch, afterFile);
|
|
if (!prev) {
|
|
console.log('No previous run of the same tier found to compare against.');
|
|
process.exit(0);
|
|
}
|
|
beforeFile = prev;
|
|
}
|
|
|
|
const beforeResult = loadResult(beforeFile);
|
|
const afterResult = loadResult(afterFile);
|
|
|
|
// Warn if different tiers
|
|
if (beforeResult.tier !== afterResult.tier) {
|
|
console.warn(`Warning: comparing different tiers (${beforeResult.tier} vs ${afterResult.tier})`);
|
|
}
|
|
|
|
// Warn on schema mismatch
|
|
if (beforeResult.schema_version !== afterResult.schema_version) {
|
|
console.warn(`Warning: schema version mismatch (${beforeResult.schema_version} vs ${afterResult.schema_version})`);
|
|
}
|
|
|
|
const comparison = compareEvalResults(beforeResult, afterResult, beforeFile, afterFile);
|
|
console.log(formatComparison(comparison));
|