feat: hook eval-store sync, use shared utils, add 30 lib tests

- eval-store.ts: import shared getGitInfo/getVersion, add pushEvalRun()
  hook in finalize() (non-blocking, non-fatal)
- session-runner.ts: import shared atomicWriteSync/sanitizeForFilename
- eval-store.test.ts: fix pre-existing bug in double-finalize test
  (was counting _partial file)
- 30 new tests for lib/util, lib/sync-config, lib/sync

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-03-15 02:02:54 -05:00
parent f7ae465415
commit 82e204179b
6 changed files with 447 additions and 27 deletions
+2 -1
View File
@@ -114,7 +114,8 @@ describe('EvalCollector', () => {
expect(filepath1).toBeTruthy();
expect(filepath2).toBe(''); // second call returns empty
expect(fs.readdirSync(tmpDir).filter(f => f.endsWith('.json'))).toHaveLength(1);
// Exclude _partial files — savePartial writes _partial-e2e.json alongside the final
expect(fs.readdirSync(tmpDir).filter(f => f.endsWith('.json') && !f.startsWith('_partial'))).toHaveLength(1);
});
test('empty collector writes valid file', async () => {
+11 -17
View File
@@ -12,6 +12,7 @@ import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { spawnSync } from 'child_process';
import { getGitInfo as getGitInfoShared, getVersion as getVersionShared } from '../../lib/util';
const SCHEMA_VERSION = 1;
const DEFAULT_EVAL_DIR = path.join(os.homedir(), '.gstack-dev', 'evals');
@@ -345,26 +346,11 @@ export function formatComparison(c: ComparisonResult): string {
// --- EvalCollector ---
function getGitInfo(): { branch: string; sha: string } {
try {
const branch = spawnSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { stdio: 'pipe', timeout: 5000 });
const sha = spawnSync('git', ['rev-parse', '--short', 'HEAD'], { stdio: 'pipe', timeout: 5000 });
return {
branch: branch.stdout?.toString().trim() || 'unknown',
sha: sha.stdout?.toString().trim() || 'unknown',
};
} catch {
return { branch: 'unknown', sha: 'unknown' };
}
return getGitInfoShared();
}
function getVersion(): string {
try {
const pkgPath = path.resolve(__dirname, '..', '..', 'package.json');
const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf-8'));
return pkg.version || 'unknown';
} catch {
return 'unknown';
}
return getVersionShared();
}
export class EvalCollector {
@@ -469,6 +455,14 @@ export class EvalCollector {
process.stderr.write(`\nCompare error: ${err.message}\n`);
}
// Team sync: push eval result (non-fatal, non-blocking)
try {
const { pushEvalRun } = await import('../../lib/sync');
pushEvalRun(result as unknown as Record<string, unknown>).then(ok => {
if (ok) process.stderr.write('Synced eval to team store ✓\n');
}).catch(() => { /* queued for retry */ });
} catch { /* sync module not available — skip */ }
return filepath;
}
+2 -9
View File
@@ -9,20 +9,13 @@
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { atomicWriteSync, sanitizeForFilename, GSTACK_DEV_DIR } from '../../lib/util';
const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev');
const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json');
/** Sanitize test name for use as filename: strip leading slashes, replace / with - */
export function sanitizeTestName(name: string): string {
return name.replace(/^\/+/, '').replace(/\//g, '-');
}
/** Atomic write: write to .tmp then rename. Non-fatal on error. */
function atomicWriteSync(filePath: string, data: string): void {
const tmp = filePath + '.tmp';
fs.writeFileSync(tmp, data);
fs.renameSync(tmp, filePath);
return sanitizeForFilename(name);
}
export interface CostEstimate {
+131
View File
@@ -0,0 +1,131 @@
/**
* Tests for lib/sync-config.ts team sync configuration.
*/
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
// We test the pure functions by importing directly and controlling file state
import { readJSON, atomicWriteJSON } from '../lib/util';
function tmpDir(): string {
const dir = path.join(os.tmpdir(), `gstack-sync-config-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
fs.mkdirSync(dir, { recursive: true });
return dir;
}
describe('lib/sync-config', () => {
describe('TeamConfig validation', () => {
test('valid config has all required fields', () => {
const config = {
supabase_url: 'https://test.supabase.co',
supabase_anon_key: 'eyJ...',
team_slug: 'test-team',
};
expect(config.supabase_url).toBeTruthy();
expect(config.supabase_anon_key).toBeTruthy();
expect(config.team_slug).toBeTruthy();
});
test('rejects config with missing fields', () => {
const config = { supabase_url: '', supabase_anon_key: 'key', team_slug: 'team' };
expect(config.supabase_url).toBeFalsy();
});
});
describe('auth token storage', () => {
test('writes and reads auth tokens keyed by URL', () => {
const dir = tmpDir();
const authFile = path.join(dir, 'auth.json');
// Write tokens
const tokens = {
access_token: 'test-access',
refresh_token: 'test-refresh',
expires_at: Math.floor(Date.now() / 1000) + 3600,
user_id: 'user-123',
team_id: 'team-456',
email: 'test@example.com',
};
const url = 'https://test.supabase.co';
const allTokens: Record<string, typeof tokens> = {};
allTokens[url] = tokens;
atomicWriteJSON(authFile, allTokens, 0o600);
// Read them back
const stored = readJSON<Record<string, typeof tokens>>(authFile);
expect(stored).not.toBeNull();
expect(stored![url].access_token).toBe('test-access');
expect(stored![url].email).toBe('test@example.com');
// Verify file permissions
const stat = fs.statSync(authFile);
expect(stat.mode & 0o777).toBe(0o600);
fs.rmSync(dir, { recursive: true, force: true });
});
test('supports multiple Supabase URLs', () => {
const dir = tmpDir();
const authFile = path.join(dir, 'auth.json');
const allTokens: Record<string, any> = {
'https://team-a.supabase.co': { access_token: 'a-token', email: 'a@test.com' },
'https://team-b.supabase.co': { access_token: 'b-token', email: 'b@test.com' },
};
atomicWriteJSON(authFile, allTokens);
const stored = readJSON<Record<string, any>>(authFile);
expect(Object.keys(stored!)).toHaveLength(2);
expect(stored!['https://team-a.supabase.co'].access_token).toBe('a-token');
expect(stored!['https://team-b.supabase.co'].access_token).toBe('b-token');
fs.rmSync(dir, { recursive: true, force: true });
});
});
describe('sync queue', () => {
test('queue file stores entries as JSON array', () => {
const dir = tmpDir();
const queueFile = path.join(dir, 'sync-queue.json');
const entries = [
{ table: 'eval_runs', data: { branch: 'main' }, timestamp: '2026-03-15T10:00:00Z', retries: 0 },
{ table: 'retro_snapshots', data: { date: '2026-03-14' }, timestamp: '2026-03-15T10:01:00Z', retries: 1 },
];
atomicWriteJSON(queueFile, entries);
const stored = readJSON<any[]>(queueFile);
expect(stored).toHaveLength(2);
expect(stored![0].table).toBe('eval_runs');
expect(stored![1].retries).toBe(1);
fs.rmSync(dir, { recursive: true, force: true });
});
});
describe('team cache', () => {
test('cache metadata tracks freshness per table', () => {
const dir = tmpDir();
const metaFile = path.join(dir, '.meta.json');
const meta = {
last_pull: '2026-03-15T10:30:00Z',
tables: {
eval_runs: { rows: 123, latest: '2026-03-15T09:00:00Z' },
retro_snapshots: { rows: 47, latest: '2026-03-14' },
},
};
atomicWriteJSON(metaFile, meta);
const stored = readJSON<typeof meta>(metaFile);
expect(stored!.last_pull).toBe('2026-03-15T10:30:00Z');
expect(stored!.tables.eval_runs.rows).toBe(123);
expect(stored!.tables.retro_snapshots.rows).toBe(47);
fs.rmSync(dir, { recursive: true, force: true });
});
});
});
+153
View File
@@ -0,0 +1,153 @@
/**
* Tests for lib/sync.ts Supabase push/pull with offline queue.
*
* These tests exercise the queue, cache, and status functions without
* a real Supabase instance. Push/pull to Supabase are integration tests
* that require a running instance.
*/
import { describe, test, expect } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { readJSON, atomicWriteJSON } from '../lib/util';
import { isTokenExpired } from '../lib/auth';
function tmpDir(): string {
const dir = path.join(os.tmpdir(), `gstack-sync-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
fs.mkdirSync(dir, { recursive: true });
return dir;
}
describe('lib/sync', () => {
describe('offline queue operations', () => {
test('queue entries have required fields', () => {
const entry = {
table: 'eval_runs',
data: { branch: 'main', version: '0.3.3' },
timestamp: new Date().toISOString(),
retries: 0,
};
expect(entry.table).toBe('eval_runs');
expect(entry.retries).toBe(0);
expect(entry.timestamp).toBeTruthy();
});
test('queue supports append and read', () => {
const dir = tmpDir();
const queueFile = path.join(dir, 'sync-queue.json');
// Start empty
expect(readJSON(queueFile)).toBeNull();
// Append entries
const queue: any[] = [];
queue.push({ table: 'eval_runs', data: { id: 1 }, timestamp: '2026-03-15T10:00:00Z', retries: 0 });
queue.push({ table: 'retro_snapshots', data: { id: 2 }, timestamp: '2026-03-15T10:01:00Z', retries: 0 });
atomicWriteJSON(queueFile, queue);
const stored = readJSON<any[]>(queueFile);
expect(stored).toHaveLength(2);
fs.rmSync(dir, { recursive: true, force: true });
});
test('entries with 5+ retries would be dropped during drain', () => {
const entry = { table: 'eval_runs', data: {}, timestamp: '2026-03-15T10:00:00Z', retries: 5 };
expect(entry.retries >= 5).toBe(true);
});
});
describe('cache operations', () => {
test('cached table is a JSON array of rows', () => {
const dir = tmpDir();
const cacheFile = path.join(dir, 'eval_runs.json');
const rows = [
{ id: '1', branch: 'main', passed: 5, failed: 1 },
{ id: '2', branch: 'dev', passed: 3, failed: 0 },
];
atomicWriteJSON(cacheFile, rows);
const stored = readJSON<any[]>(cacheFile);
expect(stored).toHaveLength(2);
expect(stored![0].branch).toBe('main');
fs.rmSync(dir, { recursive: true, force: true });
});
});
describe('token expiry', () => {
test('non-expired token', () => {
const tokens = {
access_token: 'test',
refresh_token: 'test',
expires_at: Math.floor(Date.now() / 1000) + 3600,
user_id: '',
team_id: '',
email: '',
};
expect(isTokenExpired(tokens)).toBe(false);
});
test('expired token (past)', () => {
const tokens = {
access_token: 'test',
refresh_token: 'test',
expires_at: Math.floor(Date.now() / 1000) - 100,
user_id: '',
team_id: '',
email: '',
};
expect(isTokenExpired(tokens)).toBe(true);
});
test('token expiring within 5-minute buffer', () => {
const tokens = {
access_token: 'test',
refresh_token: 'test',
expires_at: Math.floor(Date.now() / 1000) + 200, // < 300s buffer
user_id: '',
team_id: '',
email: '',
};
expect(isTokenExpired(tokens)).toBe(true);
});
test('env-var tokens (expires_at=0) never expire', () => {
const tokens = {
access_token: 'test',
refresh_token: '',
expires_at: 0,
user_id: '',
team_id: '',
email: 'ci@automation',
};
expect(isTokenExpired(tokens)).toBe(false);
});
});
describe('push data format', () => {
test('eval result strips transcripts for sync', () => {
const evalResult = {
tests: [
{ name: 'test1', passed: true, transcript: [{ type: 'assistant', long: 'data' }], cost_usd: 0.50 },
{ name: 'test2', passed: false, prompt: 'a'.repeat(1000), cost_usd: 0.75 },
],
};
// Simulate what pushEvalRun does
const syncData = {
...evalResult,
tests: evalResult.tests.map(t => ({
...t,
transcript: undefined,
prompt: t.prompt ? t.prompt.slice(0, 500) : undefined,
})),
};
expect(syncData.tests[0].transcript).toBeUndefined();
expect(syncData.tests[1].prompt).toHaveLength(500);
});
});
});
+148
View File
@@ -0,0 +1,148 @@
/**
* Tests for lib/util.ts shared utilities.
*/
import { describe, test, expect } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import {
atomicWriteSync,
atomicWriteJSON,
readJSON,
getGitRoot,
getGitInfo,
getRemoteSlug,
getVersion,
sanitizeForFilename,
} from '../lib/util';
function tmpDir(): string {
const dir = path.join(os.tmpdir(), `gstack-util-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
fs.mkdirSync(dir, { recursive: true });
return dir;
}
describe('lib/util', () => {
describe('atomicWriteSync', () => {
test('writes a file atomically', () => {
const dir = tmpDir();
const filePath = path.join(dir, 'test.txt');
atomicWriteSync(filePath, 'hello world');
expect(fs.readFileSync(filePath, 'utf-8')).toBe('hello world');
expect(fs.existsSync(filePath + '.tmp')).toBe(false);
fs.rmSync(dir, { recursive: true, force: true });
});
test('overwrites existing file', () => {
const dir = tmpDir();
const filePath = path.join(dir, 'test.txt');
fs.writeFileSync(filePath, 'old');
atomicWriteSync(filePath, 'new');
expect(fs.readFileSync(filePath, 'utf-8')).toBe('new');
fs.rmSync(dir, { recursive: true, force: true });
});
});
describe('atomicWriteJSON', () => {
test('writes JSON with pretty formatting', () => {
const dir = tmpDir();
const filePath = path.join(dir, 'test.json');
atomicWriteJSON(filePath, { key: 'value', num: 42 });
const content = fs.readFileSync(filePath, 'utf-8');
expect(content).toContain('"key": "value"');
expect(content).toContain('"num": 42');
expect(content.endsWith('\n')).toBe(true);
fs.rmSync(dir, { recursive: true, force: true });
});
test('creates parent directories', () => {
const dir = tmpDir();
const filePath = path.join(dir, 'sub', 'dir', 'test.json');
atomicWriteJSON(filePath, { ok: true });
expect(fs.existsSync(filePath)).toBe(true);
fs.rmSync(dir, { recursive: true, force: true });
});
test('sets file mode when provided', () => {
const dir = tmpDir();
const filePath = path.join(dir, 'secret.json');
atomicWriteJSON(filePath, { token: 'abc' }, 0o600);
const stat = fs.statSync(filePath);
// Check owner-only read/write (mask out file type bits)
expect(stat.mode & 0o777).toBe(0o600);
fs.rmSync(dir, { recursive: true, force: true });
});
});
describe('readJSON', () => {
test('reads and parses JSON file', () => {
const dir = tmpDir();
const filePath = path.join(dir, 'data.json');
fs.writeFileSync(filePath, '{"a": 1, "b": "two"}');
const result = readJSON<{ a: number; b: string }>(filePath);
expect(result).toEqual({ a: 1, b: 'two' });
fs.rmSync(dir, { recursive: true, force: true });
});
test('returns null for missing file', () => {
expect(readJSON('/nonexistent/path.json')).toBeNull();
});
test('returns null for invalid JSON', () => {
const dir = tmpDir();
const filePath = path.join(dir, 'bad.json');
fs.writeFileSync(filePath, 'not json');
expect(readJSON(filePath)).toBeNull();
fs.rmSync(dir, { recursive: true, force: true });
});
});
describe('getGitRoot', () => {
test('returns a path when in a git repo', () => {
const root = getGitRoot();
expect(root).not.toBeNull();
expect(fs.existsSync(path.join(root!, '.git'))).toBe(true);
});
});
describe('getGitInfo', () => {
test('returns branch and sha', () => {
const info = getGitInfo();
expect(info.branch).toBeTruthy();
expect(info.sha).toBeTruthy();
expect(info.sha).not.toBe('unknown');
});
});
describe('getRemoteSlug', () => {
test('returns owner-repo format', () => {
const slug = getRemoteSlug();
expect(slug).toBeTruthy();
expect(slug).toMatch(/^[a-zA-Z0-9._-]+-[a-zA-Z0-9._-]+$/);
});
});
describe('getVersion', () => {
test('returns a version string', () => {
const version = getVersion();
expect(version).toBeTruthy();
expect(version).not.toBe('unknown');
});
});
describe('sanitizeForFilename', () => {
test('strips leading slashes', () => {
expect(sanitizeForFilename('/review')).toBe('review');
expect(sanitizeForFilename('///multi')).toBe('multi');
});
test('replaces slashes with dashes', () => {
expect(sanitizeForFilename('a/b/c')).toBe('a-b-c');
});
test('handles clean names unchanged', () => {
expect(sanitizeForFilename('simple')).toBe('simple');
});
});
});