mirror of
https://github.com/garrytan/gstack.git
synced 2026-05-06 21:46:40 +02:00
feat: add SHA-based eval caching with EVAL_CACHE=0 bypass
Cache at ~/.gstack/eval-cache/{suite}/{sha}.json. Compute cache keys
from source file contents + test input via Bun.CryptoHasher SHA256.
Supports read/write/stats/clear/verify operations. EVAL_CACHE=0
skips reads for force-rerun. 16 tests including corrupt JSON handling.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
@@ -0,0 +1,213 @@
|
||||
/**
|
||||
* SHA-based eval caching.
|
||||
*
|
||||
* Cache path: ~/.gstack/eval-cache/{suite}/{sha}.json
|
||||
*
|
||||
* Caches eval results keyed by a SHA256 hash of source files + test input.
|
||||
* Supports EVAL_CACHE=0 to skip reads (always re-run).
|
||||
*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { atomicWriteJSON, readJSON } from './util';
|
||||
|
||||
const CACHE_VERSION = 1;
|
||||
|
||||
/** Resolve cache dir lazily so GSTACK_STATE_DIR env overrides work in tests. */
|
||||
function getCacheDir(): string {
|
||||
const stateDir = process.env.GSTACK_STATE_DIR || require('path').join(require('os').homedir(), '.gstack');
|
||||
return path.join(stateDir, 'eval-cache');
|
||||
}
|
||||
|
||||
// --- Cache key ---
|
||||
|
||||
/**
|
||||
* Compute a cache key from source file contents + test input.
|
||||
* Returns first 16 hex chars of SHA256.
|
||||
*/
|
||||
export function computeCacheKey(sourceFiles: string[], testInput: string): string {
|
||||
const hasher = new Bun.CryptoHasher('sha256');
|
||||
for (const file of sourceFiles.sort()) {
|
||||
try {
|
||||
hasher.update(fs.readFileSync(file));
|
||||
} catch (err: any) {
|
||||
throw new Error(`Cache key: cannot read source file "${file}": ${err.message}`);
|
||||
}
|
||||
}
|
||||
hasher.update(testInput);
|
||||
return hasher.digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
// --- Read / Write ---
|
||||
|
||||
function cachePath(suite: string, key: string): string {
|
||||
return path.join(getCacheDir(), suite, `${key}.json`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a cached value. Returns null on miss, corrupt data, or if EVAL_CACHE=0.
|
||||
*/
|
||||
export function cacheRead(suite: string, key: string): unknown | null {
|
||||
if (process.env.EVAL_CACHE === '0') return null;
|
||||
const filePath = cachePath(suite, key);
|
||||
const envelope = readJSON<{ _cache_version: number; data: unknown }>(filePath);
|
||||
if (!envelope || envelope._cache_version !== CACHE_VERSION) return null;
|
||||
return envelope.data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a value to cache. Atomic write with metadata envelope.
|
||||
*/
|
||||
export function cacheWrite(
|
||||
suite: string,
|
||||
key: string,
|
||||
data: unknown,
|
||||
meta?: Record<string, unknown>,
|
||||
): void {
|
||||
const filePath = cachePath(suite, key);
|
||||
const envelope = {
|
||||
_cache_version: CACHE_VERSION,
|
||||
_cached_at: new Date().toISOString(),
|
||||
_suite: suite,
|
||||
...meta,
|
||||
data,
|
||||
};
|
||||
try {
|
||||
atomicWriteJSON(filePath, envelope);
|
||||
} catch (err: any) {
|
||||
throw new Error(`Cache write failed for "${filePath}": ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Management ---
|
||||
|
||||
interface SuiteStats {
|
||||
name: string;
|
||||
entries: number;
|
||||
size_bytes: number;
|
||||
oldest: string;
|
||||
newest: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cache statistics. If suite is provided, stats for that suite only.
|
||||
*/
|
||||
export function cacheStats(suite?: string): { suites: SuiteStats[] } {
|
||||
const suites: SuiteStats[] = [];
|
||||
|
||||
let dirNames: string[];
|
||||
try {
|
||||
dirNames = suite ? [suite] : fs.readdirSync(getCacheDir());
|
||||
} catch {
|
||||
return { suites: [] };
|
||||
}
|
||||
|
||||
for (const name of dirNames) {
|
||||
const suiteDir = path.join(getCacheDir(), name);
|
||||
try {
|
||||
const stat = fs.statSync(suiteDir);
|
||||
if (!stat.isDirectory()) continue;
|
||||
} catch { continue; }
|
||||
|
||||
let files: string[];
|
||||
try {
|
||||
files = fs.readdirSync(suiteDir).filter(f => f.endsWith('.json'));
|
||||
} catch { continue; }
|
||||
|
||||
if (files.length === 0) {
|
||||
suites.push({ name, entries: 0, size_bytes: 0, oldest: '', newest: '' });
|
||||
continue;
|
||||
}
|
||||
|
||||
let totalSize = 0;
|
||||
let oldest = '';
|
||||
let newest = '';
|
||||
|
||||
for (const file of files) {
|
||||
try {
|
||||
const fileStat = fs.statSync(path.join(suiteDir, file));
|
||||
totalSize += fileStat.size;
|
||||
const mtime = fileStat.mtime.toISOString();
|
||||
if (!oldest || mtime < oldest) oldest = mtime;
|
||||
if (!newest || mtime > newest) newest = mtime;
|
||||
} catch { continue; }
|
||||
}
|
||||
|
||||
suites.push({ name, entries: files.length, size_bytes: totalSize, oldest, newest });
|
||||
}
|
||||
|
||||
return { suites };
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear cache entries. If suite is provided, clears only that suite.
|
||||
* Returns count of deleted files.
|
||||
*/
|
||||
export function cacheClear(suite?: string): { deleted: number } {
|
||||
let deleted = 0;
|
||||
|
||||
let dirNames: string[];
|
||||
try {
|
||||
dirNames = suite ? [suite] : fs.readdirSync(getCacheDir());
|
||||
} catch {
|
||||
return { deleted: 0 };
|
||||
}
|
||||
|
||||
for (const name of dirNames) {
|
||||
const suiteDir = path.join(getCacheDir(), name);
|
||||
try {
|
||||
const files = fs.readdirSync(suiteDir).filter(f => f.endsWith('.json'));
|
||||
for (const file of files) {
|
||||
fs.unlinkSync(path.join(suiteDir, file));
|
||||
deleted++;
|
||||
}
|
||||
// Remove empty directory
|
||||
try { fs.rmdirSync(suiteDir); } catch { /* not empty or doesn't exist */ }
|
||||
} catch { continue; }
|
||||
}
|
||||
|
||||
return { deleted };
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify cache integrity. Checks that all cache files are valid JSON
|
||||
* with the correct cache version.
|
||||
*/
|
||||
export function cacheVerify(suite?: string): { valid: number; invalid: number; errors: string[] } {
|
||||
let valid = 0;
|
||||
let invalid = 0;
|
||||
const errors: string[] = [];
|
||||
|
||||
let dirNames: string[];
|
||||
try {
|
||||
dirNames = suite ? [suite] : fs.readdirSync(getCacheDir());
|
||||
} catch {
|
||||
return { valid: 0, invalid: 0, errors: [] };
|
||||
}
|
||||
|
||||
for (const name of dirNames) {
|
||||
const suiteDir = path.join(getCacheDir(), name);
|
||||
let files: string[];
|
||||
try {
|
||||
files = fs.readdirSync(suiteDir).filter(f => f.endsWith('.json'));
|
||||
} catch { continue; }
|
||||
|
||||
for (const file of files) {
|
||||
const filePath = path.join(suiteDir, file);
|
||||
try {
|
||||
const content = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
|
||||
if (content._cache_version !== CACHE_VERSION) {
|
||||
invalid++;
|
||||
errors.push(`${name}/${file}: wrong cache version (${content._cache_version})`);
|
||||
} else {
|
||||
valid++;
|
||||
}
|
||||
} catch (err: any) {
|
||||
invalid++;
|
||||
errors.push(`${name}/${file}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { valid, invalid, errors };
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
/**
|
||||
* Tests for lib/eval-cache.ts — SHA-based eval caching.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import {
|
||||
computeCacheKey,
|
||||
cacheRead,
|
||||
cacheWrite,
|
||||
cacheStats,
|
||||
cacheClear,
|
||||
cacheVerify,
|
||||
} from '../lib/eval-cache';
|
||||
|
||||
describe('lib/eval-cache', () => {
|
||||
let origStateDir: string | undefined;
|
||||
let testDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
origStateDir = process.env.GSTACK_STATE_DIR;
|
||||
const unique = `${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
||||
testDir = path.join(os.tmpdir(), `gstack-cache-test-${unique}`);
|
||||
fs.mkdirSync(testDir, { recursive: true });
|
||||
process.env.GSTACK_STATE_DIR = testDir;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (origStateDir === undefined) delete process.env.GSTACK_STATE_DIR;
|
||||
else process.env.GSTACK_STATE_DIR = origStateDir;
|
||||
fs.rmSync(testDir, { recursive: true, force: true });
|
||||
delete process.env.EVAL_CACHE;
|
||||
});
|
||||
|
||||
describe('computeCacheKey', () => {
|
||||
test('produces deterministic 16-char hex key', () => {
|
||||
const srcDir = path.join(testDir, 'src');
|
||||
fs.mkdirSync(srcDir, { recursive: true });
|
||||
const file = path.join(srcDir, 'test.ts');
|
||||
fs.writeFileSync(file, 'const x = 1;');
|
||||
|
||||
const key1 = computeCacheKey([file], 'test input');
|
||||
const key2 = computeCacheKey([file], 'test input');
|
||||
expect(key1).toBe(key2);
|
||||
expect(key1.length).toBe(16);
|
||||
expect(key1).toMatch(/^[0-9a-f]+$/);
|
||||
});
|
||||
|
||||
test('different inputs produce different keys', () => {
|
||||
const srcDir = path.join(testDir, 'src');
|
||||
fs.mkdirSync(srcDir, { recursive: true });
|
||||
const file = path.join(srcDir, 'test.ts');
|
||||
fs.writeFileSync(file, 'const x = 1;');
|
||||
|
||||
const key1 = computeCacheKey([file], 'input A');
|
||||
const key2 = computeCacheKey([file], 'input B');
|
||||
expect(key1).not.toBe(key2);
|
||||
});
|
||||
|
||||
test('throws on missing source file', () => {
|
||||
expect(() => computeCacheKey(['/nonexistent/file.ts'], 'test'))
|
||||
.toThrow('cannot read source file');
|
||||
});
|
||||
});
|
||||
|
||||
describe('cacheRead / cacheWrite', () => {
|
||||
test('write then read round-trips data', () => {
|
||||
const data = { result: 'ok', score: 42 };
|
||||
cacheWrite('test-suite', 'abc123', data);
|
||||
const read = cacheRead('test-suite', 'abc123');
|
||||
expect(read).toEqual(data);
|
||||
});
|
||||
|
||||
test('read returns null on cache miss', () => {
|
||||
const read = cacheRead('test-suite', 'nonexistent');
|
||||
expect(read).toBeNull();
|
||||
});
|
||||
|
||||
test('read returns null when EVAL_CACHE=0', () => {
|
||||
cacheWrite('test-suite', 'abc123', { data: 'cached' });
|
||||
process.env.EVAL_CACHE = '0';
|
||||
const read = cacheRead('test-suite', 'abc123');
|
||||
expect(read).toBeNull();
|
||||
});
|
||||
|
||||
test('read returns null for corrupt JSON', () => {
|
||||
const cacheDir = path.join(testDir, 'eval-cache', 'test-suite');
|
||||
fs.mkdirSync(cacheDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(cacheDir, 'corrupt.json'), 'not valid json{{{');
|
||||
const read = cacheRead('test-suite', 'corrupt');
|
||||
expect(read).toBeNull();
|
||||
});
|
||||
|
||||
test('read returns null for wrong cache version', () => {
|
||||
const cacheDir = path.join(testDir, 'eval-cache', 'test-suite');
|
||||
fs.mkdirSync(cacheDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(cacheDir, 'old.json'), JSON.stringify({
|
||||
_cache_version: 999,
|
||||
data: { stale: true },
|
||||
}));
|
||||
const read = cacheRead('test-suite', 'old');
|
||||
expect(read).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('cacheStats', () => {
|
||||
test('returns empty for nonexistent cache', () => {
|
||||
const stats = cacheStats();
|
||||
expect(stats.suites).toEqual([]);
|
||||
});
|
||||
|
||||
test('returns stats after writes', () => {
|
||||
cacheWrite('suite-a', 'key1', { a: 1 });
|
||||
cacheWrite('suite-a', 'key2', { a: 2 });
|
||||
cacheWrite('suite-b', 'key1', { b: 1 });
|
||||
|
||||
const stats = cacheStats();
|
||||
expect(stats.suites.length).toBe(2);
|
||||
const suiteA = stats.suites.find(s => s.name === 'suite-a');
|
||||
expect(suiteA).toBeDefined();
|
||||
expect(suiteA!.entries).toBe(2);
|
||||
expect(suiteA!.size_bytes).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('filters by suite name', () => {
|
||||
cacheWrite('suite-a', 'key1', { a: 1 });
|
||||
cacheWrite('suite-b', 'key1', { b: 1 });
|
||||
|
||||
const stats = cacheStats('suite-a');
|
||||
expect(stats.suites.length).toBe(1);
|
||||
expect(stats.suites[0].name).toBe('suite-a');
|
||||
});
|
||||
});
|
||||
|
||||
describe('cacheClear', () => {
|
||||
test('clears all entries', () => {
|
||||
cacheWrite('suite-a', 'key1', { a: 1 });
|
||||
cacheWrite('suite-b', 'key1', { b: 1 });
|
||||
|
||||
const result = cacheClear();
|
||||
expect(result.deleted).toBe(2);
|
||||
|
||||
const stats = cacheStats();
|
||||
expect(stats.suites.length).toBe(0);
|
||||
});
|
||||
|
||||
test('clears specific suite', () => {
|
||||
cacheWrite('suite-a', 'key1', { a: 1 });
|
||||
cacheWrite('suite-b', 'key1', { b: 1 });
|
||||
|
||||
cacheClear('suite-a');
|
||||
expect(cacheRead('suite-a', 'key1')).toBeNull();
|
||||
expect(cacheRead('suite-b', 'key1')).toEqual({ b: 1 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('cacheVerify', () => {
|
||||
test('reports valid entries', () => {
|
||||
cacheWrite('suite-a', 'key1', { a: 1 });
|
||||
|
||||
const result = cacheVerify();
|
||||
expect(result.valid).toBe(1);
|
||||
expect(result.invalid).toBe(0);
|
||||
expect(result.errors).toEqual([]);
|
||||
});
|
||||
|
||||
test('detects corrupt entries', () => {
|
||||
cacheWrite('suite-a', 'key1', { a: 1 });
|
||||
|
||||
// Write corrupt file alongside valid one
|
||||
const cacheDir = path.join(testDir, 'eval-cache', 'suite-a');
|
||||
fs.writeFileSync(path.join(cacheDir, 'bad.json'), 'not json');
|
||||
|
||||
const result = cacheVerify();
|
||||
expect(result.valid).toBe(1);
|
||||
expect(result.invalid).toBe(1);
|
||||
expect(result.errors.length).toBe(1);
|
||||
});
|
||||
|
||||
test('detects wrong version', () => {
|
||||
const cacheDir = path.join(testDir, 'eval-cache', 'suite-a');
|
||||
fs.mkdirSync(cacheDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(cacheDir, 'old.json'), JSON.stringify({ _cache_version: 0 }));
|
||||
|
||||
const result = cacheVerify();
|
||||
expect(result.invalid).toBe(1);
|
||||
expect(result.errors[0]).toContain('wrong cache version');
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user