CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-17 10:25:33 +02:00
Code Issues Packages Projects Releases Wiki Activity

refactor(evals): shared partial-run predicate + finalized-run lookup

Browse Source 
isPartialEval(data, filename) is the one place that decides what counts
as an in-progress accumulator (the _partial flag OR a _partial-prefixed
filename), and findLatestFinalizedRun(evalDir, tier) is the one place
that finds the newest real run — scanning the eval dir plus one level of
shards/<slug>/ subdirs, where the sharded paid runner points each
shard's collector. skill-budget-regression.test.ts's hand-rolled
findLatestRun (flag-blind: a flagged-but-renamed accumulator passed its
name check) is replaced by the shared helper.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
(cherry picked from commit b55fcf6966366fd21a8cdc46de61aab6e1b1d100)
This commit is contained in:
Garry Tan
2026-08-12 15:31:47 -07:00
parent 39e8e789bf
commit 553477bead
3 changed files with 164 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/helpers/eval-store.test.ts
+89
View File
@@ -6,6 +6,9 @@ import {
EvalCollector,
extractToolSummary,
findPreviousRun,
findLatestFinalizedRun,
isPartialEval,
listEvalJsonFiles,
compareEvalResults,
formatComparison,
generateCommentary,
@@ -346,6 +349,92 @@ describe('findPreviousRun', () => {
});
});
// --- isPartialEval tests ---
describe('isPartialEval', () => {
test('flag set but file renamed — still partial', () => {
expect(isPartialEval({ _partial: true }, 'renamed-to-look-final.json')).toBe(true);
});
test('name matches but no flag — still partial', () => {
expect(isPartialEval({}, '_partial-e2e.json')).toBe(true);
expect(isPartialEval(null, path.join('/some/dir', '_partial-e2e.json'))).toBe(true);
});
test('finalized run is not partial', () => {
expect(isPartialEval(makeResult(), '0.3.6-main-e2e-20260314-100000.json')).toBe(false);
});
});
// --- listEvalJsonFiles / findLatestFinalizedRun tests ---
describe('findLatestFinalizedRun', () => {
test('listEvalJsonFiles recurses exactly one shards/*/ level', () => {
fs.writeFileSync(path.join(tmpDir, 'flat.json'), '{}');
const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa');
fs.mkdirSync(shardDir, { recursive: true });
fs.writeFileSync(path.join(shardDir, 'sharded.json'), '{}');
// Nested one level deeper than the contract — must NOT be picked up.
const tooDeep = path.join(shardDir, 'shards', 'nested');
fs.mkdirSync(tooDeep, { recursive: true });
fs.writeFileSync(path.join(tooDeep, 'too-deep.json'), '{}');
fs.writeFileSync(path.join(tmpDir, 'not-json.txt'), '');
const files = listEvalJsonFiles(tmpDir).map(f => path.basename(f)).sort();
expect(files).toEqual(['flat.json', 'sharded.json']);
});
test('finds the newest finalized run across flat dir and shard subdirs', () => {
fs.writeFileSync(
path.join(tmpDir, '0.3.5-main-e2e-20260312-100000.json'),
JSON.stringify(makeResult({ timestamp: '2026-03-12T10:00:00Z' })),
);
const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa');
fs.mkdirSync(shardDir, { recursive: true });
fs.writeFileSync(
path.join(shardDir, '0.3.6-main-e2e-20260314-100000.json'),
JSON.stringify(makeResult({ timestamp: '2026-03-14T10:00:00Z' })),
);
const latest = findLatestFinalizedRun(tmpDir, 'e2e');
expect(latest?.filepath).toContain('skill-e2e-qa');
expect(latest?.result.timestamp).toBe('2026-03-14T10:00:00Z');
});
test('skips partials by flag and by name, filters by tier', () => {
// Newest by timestamp, but partial by flag under a shard dir.
const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa');
fs.mkdirSync(shardDir, { recursive: true });
fs.writeFileSync(
path.join(shardDir, 'flagged.json'),
JSON.stringify(makeResult({ timestamp: '2026-03-16T10:00:00Z', _partial: true })),
);
// Partial by name only.
fs.writeFileSync(
path.join(tmpDir, '_partial-e2e.json'),
JSON.stringify(makeResult({ timestamp: '2026-03-15T10:00:00Z' })),
);
// Wrong tier.
fs.writeFileSync(
path.join(tmpDir, '0.3.6-main-llm-judge-20260317-100000.json'),
JSON.stringify(makeResult({ tier: 'llm-judge', timestamp: '2026-03-17T10:00:00Z' })),
);
// The genuine baseline.
fs.writeFileSync(
path.join(tmpDir, '0.3.5-main-e2e-20260312-100000.json'),
JSON.stringify(makeResult({ timestamp: '2026-03-12T10:00:00Z' })),
);
const latest = findLatestFinalizedRun(tmpDir, 'e2e');
expect(latest?.result.timestamp).toBe('2026-03-12T10:00:00Z');
});
test('returns null for missing dir or no finalized runs', () => {
expect(findLatestFinalizedRun('/nonexistent/path', 'e2e')).toBeNull();
expect(findLatestFinalizedRun(tmpDir, 'e2e')).toBeNull();
});
});
// --- compareEvalResults tests ---
describe('compareEvalResults', () => {