CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-18 19:02:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor(evals): shared partial-run predicate + finalized-run lookup

Browse Source 
isPartialEval(data, filename) is the one place that decides what counts
as an in-progress accumulator (the _partial flag OR a _partial-prefixed
filename), and findLatestFinalizedRun(evalDir, tier) is the one place
that finds the newest real run — scanning the eval dir plus one level of
shards/<slug>/ subdirs, where the sharded paid runner points each
shard's collector. skill-budget-regression.test.ts's hand-rolled
findLatestRun (flag-blind: a flagged-but-renamed accumulator passed its
name check) is replaced by the shared helper.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
(cherry picked from commit b55fcf6966366fd21a8cdc46de61aab6e1b1d100)
This commit is contained in:
Garry Tan
2026-08-12 15:31:47 -07:00
parent 39e8e789bf
commit 553477bead
3 changed files with 164 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/skill-budget-regression.test.ts
+3 -36
View File
@@ -27,10 +27,10 @@
import { describe, test } from 'bun:test';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import {
getProjectEvalDir,
findPreviousRun,
findLatestFinalizedRun,
compareEvalResults,
assertNoBudgetRegression,
type EvalResult,
@@ -72,42 +72,9 @@ function currentGitBranch(): string {
}
}
interface LatestRun {
filepath: string;
result: EvalResult;
}
/** Find the most recent finalized (non-_partial) eval file for a tier. */
function findLatestRun(evalDir: string, tier: 'e2e' | 'llm-judge'): LatestRun | null {
let entries: string[];
try {
entries = fs.readdirSync(evalDir);
} catch {
return null;
}
const candidates: Array<{ filepath: string; timestamp: string }> = [];
for (const f of entries) {
if (!f.endsWith('.json')) continue;
if (f.startsWith('_partial')) continue;
const fullPath = path.join(evalDir, f);
try {
const data = JSON.parse(fs.readFileSync(fullPath, 'utf-8')) as EvalResult;
if (data.tier !== tier) continue;
candidates.push({ filepath: fullPath, timestamp: data.timestamp ?? '' });
} catch { /* ignore corrupt */ }
}
if (candidates.length === 0) return null;
candidates.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
const top = candidates[0]!;
return {
filepath: top.filepath,
result: JSON.parse(fs.readFileSync(top.filepath, 'utf-8')) as EvalResult,
};
}
function checkTier(tier: 'e2e' | 'llm-judge'): void {
const evalDir = getProjectEvalDir();
const latest = findLatestRun(evalDir, tier);
const latest = findLatestFinalizedRun(evalDir, tier);
if (!latest) {
// eslint-disable-next-line no-console
console.log(`[budget-regression:${tier}] no current run in ${evalDir} — skipping`);
@@ -165,7 +132,7 @@ function checkTier(tier: 'e2e' | 'llm-judge'): void {
/** Enforce a hard dollar cap on per-run eval cost. */
function checkHardCap(tier: 'e2e' | 'llm-judge'): void {
const evalDir = getProjectEvalDir();
const latest = findLatestRun(evalDir, tier);
const latest = findLatestFinalizedRun(evalDir, tier);
if (!latest) return;
const cap = TIER_CAPS[tier];
const cost = latest.result.total_cost_usd;